feat(metrics): add ability to use OTLP source

SumoLogic · Mar 21, 2023 · fcb9561 · fcb9561
1 parent 4b80aa2
commit fcb9561
Show file tree

Hide file tree

Showing 22 changed files with 222 additions and 5 deletions.
diff --git a/.changelog/2949.added.txt b/.changelog/2949.added.txt
@@ -0,0 +1 @@
+feat(metrics): add ability to use OTLP source
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -103,6 +103,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.metrics.remoteWriteProxy.podAnnotations`                              | Additional annotations for for the remote write proxy container.                                                                                                                                                                                                                                                                                                     | `{}`                                                                                                                                                                                 |
 | `sumologic.metrics.remoteWriteProxy.config.port`                                 | Port on which remote write proxy is going to be exposed                                                                                                                                                                                                                                                                                                              | `8080`                                                                                                                                                                               |
 | `sumologic.metrics.serviceMonitors`                                              | Configuration of Sumo Logic Kubernetes Collection components serviceMonitors                                                                                                                                                                                                                                                                                         | See [values.yaml]                                                                                                                                                                    |
+| `sumologic.metrics.sourceType`                                                   | The type of the Sumo Logic source being used for metrics ingestion. Can be `http` or `otlp`.                                                                                                                                                                                                                                                                         | `http`                                                                                                                                                                               |
 | `sumologic.traces.enabled`                                                       | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_                                                                                                                                                                                            | `true`                                                                                                                                                                               |
 | `sumologic.traces.spans_per_request`                                             | Maximum number of spans sent in single batch                                                                                                                                                                                                                                                                                                                         | `100`                                                                                                                                                                                |
 | `sumologic.envFromSecret`                                                        | If enabled, accessId and accessKey will be sourced from Secret Name given. Be sure to include at least the following env variables in your secret (1) SUMOLOGIC_ACCESSID, (2) SUMOLOGIC_ACCESSKEY                                                                                                                                                                    | `sumo-api-secret`                                                                                                                                                                    |

diff --git a/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml b/deploy/helm/sumologic/conf/metrics/otelcol/config.yaml
@@ -34,8 +34,8 @@ exporters:
   ## Configuration for Sumo Logic Exporter
   ## ref: https://github.com/SumoLogic/sumologic-otel-collector/blob/main/pkg/exporter/sumologicexporter
   sumologic/default:
-    metric_format: prometheus
-    endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}
+    metric_format: {{ include "metrics.otelcol.exporter.format" . }}
+    endpoint: {{ include "metrics.otelcol.exporter.endpoint" . }}
     ## Configuration for sending queue
     ## ref: https://github.com/open-telemetry/opentelemetry-collector/tree/release/v0.37.x/exporter/exporterhelper#configuration
     sending_queue:
@@ -50,6 +50,7 @@ exporters:
     max_request_body_size: 16_777_216  # 16 MB before compression
     ## set timeout to 30s due to big requests
     timeout: 30s
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
   sumologic/apiserver:
     metric_format: prometheus
     endpoint: ${SUMO_ENDPOINT_APISERVER_METRICS_SOURCE}
@@ -141,6 +142,7 @@ exporters:
     max_request_body_size: 16_777_216  # 16 MB before compression
     ## set timeout to 30s due to big requests
     timeout: 30s
+{{- end }}
 processors:
   ## Configuration for Metrics Transform Processor
   ## ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/processor/metricstransformprocessor
@@ -197,6 +199,7 @@ processors:
     attributes:
       - action: delete
         key: k8s.pod.pod_name
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
   ## NOTE: below listed rules could be simplified if routingprocessor
   ## supports regex matching. At this point we could group route entries
   ## going to the same set of exporters.
@@ -244,6 +247,7 @@ processors:
       - value: /prometheus.metrics.state
         exporters:
           - sumologic/state
+{{- end }}
 
   ## Configuration for Memory Limiter Processor
   ## The memory_limiter processor is used to prevent out of memory situations on the collector.
@@ -333,14 +337,18 @@ service:
         - resource/delete_source_metadata
         - sumologic_schema
         - batch
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
         - routing
+{{- end }}
       exporters:
         - sumologic/default
+{{- if eq .Values.sumologic.metrics.sourceType "http" }}
         - sumologic/apiserver
         - sumologic/control_plane
         - sumologic/controller
         - sumologic/kubelet
         - sumologic/node
         - sumologic/scheduler
         - sumologic/state
+{{- end }}
 
diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl
@@ -1626,6 +1626,36 @@ Return the log format for the Sumologic exporter for container logs.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the metrics format for the default Sumologic exporter for metrics.
+
+'{{ include "metrics.otelcol.exporter.format" . }}'
+*/}}
+{{- define "metrics.otelcol.exporter.format" -}}
+{{- if eq .Values.sumologic.metrics.sourceType "http" -}}
+{{- "prometheus" -}}
+{{- else if eq .Values.sumologic.metrics.sourceType "otlp" -}}
+{{- "otlp" -}}
+{{- else -}}
+{{- fail "`sumologic.metrics.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the endpoint for the default Sumologic exporter for metrics.
+
+'{{ include "metrics.otelcol.exporter.endpoint" . }}'
+*/}}
+{{- define "metrics.otelcol.exporter.endpoint" -}}
+{{- if eq .Values.sumologic.metrics.sourceType "http" -}}
+{{- "${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE}" -}}
+{{- else if eq .Values.sumologic.metrics.sourceType "otlp" -}}
+{{- "${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}" -}}
+{{- else -}}
+{{- fail "`sumologic.metrics.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
 {{/*
 Return otlp or none for Instrumentation resource exporters configuration.
 

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -164,6 +164,11 @@ sumologic:
         default:
           name: (default-metrics)
           config-name: endpoint-metrics
+        default-otlp:
+          name: default-metrics-otlp
+          config-name: endpoint-metrics-otlp
+          properties:
+            content_type: Otlp
         apiserver:
           name: apiserver-metrics
           config-name: endpoint-metrics-apiserver
@@ -580,6 +585,8 @@ sumologic:
           matchLabels:
             operated-prometheus: "true"
 
+    sourceType: http
+
   ### Traces configuration
   ## Set the enabled flag to false to disable traces ingestion.
   traces:

diff --git a/tests/helm/metrics_test.go b/tests/helm/metrics_test.go
@@ -4,6 +4,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
@@ -136,3 +137,44 @@ func TestMetadataMetricsOtelConfigExtraProcessors(t *testing.T) {
 
 	require.Equal(t, expectedPipelineValue, otelConfig.Service.Pipelines.Metrics.Processors)
 }
+
+func TestMetadataSourceTypeOTLP(t *testing.T) {
+	t.Parallel()
+	templatePath := "templates/metrics/otelcol/configmap.yaml"
+
+	type OtelConfig struct {
+		Exporters struct {
+			Default struct {
+				MetricFormat string `yaml:"metric_format"`
+				Endpoint     string
+			} `yaml:"sumologic/default"`
+			Rest map[string]interface{} `yaml:",inline"`
+		}
+		Processors map[string]interface{}
+		Service    struct {
+			Pipelines struct {
+				Metrics struct {
+					Processors []string `yaml:"processors"`
+					Exporters  []string `yaml:"exporters"`
+				}
+			}
+		}
+	}
+
+	var otelConfig OtelConfig
+	valuesYaml := `
+sumologic:
+  metrics:
+    sourceType: otlp
+`
+	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+
+	assert.Equal(t, otelConfig.Exporters.Default.MetricFormat, "otlp")
+	assert.Equal(t, otelConfig.Exporters.Default.Endpoint, "${SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE}")
+	assert.Len(t, otelConfig.Exporters.Rest, 0)
+	assert.NotContains(t, otelConfig.Processors, "routing")
+	assert.NotContains(t, otelConfig.Service.Pipelines.Metrics.Processors, "routing")
+	assert.Equal(t, otelConfig.Service.Pipelines.Metrics.Exporters, []string{"sumologic/default"})
+}
diff --git a/tests/helm/testdata/goldenfile/metadata_metrics_otc_statefulset/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_metrics_otc_statefulset/basic.output.yaml
@@ -142,6 +142,11 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-metrics
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_METRICS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-metrics-otlp
             - name: SUMO_ENDPOINT_KUBELET_METRICS_SOURCE
               valueFrom:
                 secretKeyRef:
@@ -164,6 +169,7 @@ spec:
                   key: endpoint-metrics-kube-state
             - name: NO_PROXY
               value: kubernetes.default.svc
+
             - name: VALUE_FROM_SECRET
               valueFrom:
                 secretKeyRef:

diff --git a/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml
@@ -288,6 +288,7 @@ data:
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
       default_metrics_source                      = "(default-metrics)"
+      default_otlp_metrics_source                 = "default-metrics-otlp"
       kubelet_metrics_source                      = "kubelet-metrics"
       node_metrics_source                         = "node-exporter-metrics"
       scheduler_metrics_source                    = "kube-scheduler-metrics"
@@ -435,6 +436,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_metrics_source" {
+        name         = local.default_otlp_metrics_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "kubelet_metrics_source" {
         name         = local.kubelet_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -519,6 +526,7 @@ data:
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
         endpoint-metrics                          = sumologic_http_source.default_metrics_source.url
+        endpoint-metrics-otlp                     = sumologic_http_source.default_otlp_metrics_source.url
         endpoint-metrics-kubelet                  = sumologic_http_source.kubelet_metrics_source.url
         endpoint-metrics-node-exporter            = sumologic_http_source.node_metrics_source.url
         endpoint-metrics-kube-scheduler           = sumologic_http_source.scheduler_metrics_source.url
@@ -662,6 +670,7 @@ data:
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"
     terraform import sumologic_http_source.default_metrics_source "${COLLECTOR_NAME}/(default-metrics)"
+    terraform import sumologic_http_source.default_otlp_metrics_source "${COLLECTOR_NAME}/default-metrics-otlp"
     terraform import sumologic_http_source.kubelet_metrics_source "${COLLECTOR_NAME}/kubelet-metrics"
     terraform import sumologic_http_source.node_metrics_source "${COLLECTOR_NAME}/node-exporter-metrics"
     terraform import sumologic_http_source.scheduler_metrics_source "${COLLECTOR_NAME}/kube-scheduler-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml
@@ -288,6 +288,7 @@ data:
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
       default_metrics_source                      = "(default-metrics)"
+      default_otlp_metrics_source                 = "default-metrics-otlp"
       kubelet_metrics_source                      = "kubelet-metrics"
       node_metrics_source                         = "node-exporter-metrics"
       scheduler_metrics_source                    = "kube-scheduler-metrics"
@@ -436,6 +437,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_metrics_source" {
+        name         = local.default_otlp_metrics_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "kubelet_metrics_source" {
         name         = local.kubelet_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -475,6 +482,7 @@ data:
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
         endpoint-metrics                          = sumologic_http_source.default_metrics_source.url
+        endpoint-metrics-otlp                     = sumologic_http_source.default_otlp_metrics_source.url
         endpoint-metrics-kubelet                  = sumologic_http_source.kubelet_metrics_source.url
         endpoint-metrics-node-exporter            = sumologic_http_source.node_metrics_source.url
         endpoint-metrics-kube-scheduler           = sumologic_http_source.scheduler_metrics_source.url
@@ -617,6 +625,7 @@ data:
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"
     terraform import sumologic_http_source.default_metrics_source "${COLLECTOR_NAME}/(default-metrics)"
+    terraform import sumologic_http_source.default_otlp_metrics_source "${COLLECTOR_NAME}/default-metrics-otlp"
     terraform import sumologic_http_source.kubelet_metrics_source "${COLLECTOR_NAME}/kubelet-metrics"
     terraform import sumologic_http_source.node_metrics_source "${COLLECTOR_NAME}/node-exporter-metrics"
     terraform import sumologic_http_source.scheduler_metrics_source "${COLLECTOR_NAME}/kube-scheduler-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/default.output.yaml b/tests/helm/testdata/goldenfile/terraform/default.output.yaml
@@ -288,6 +288,7 @@ data:
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
       default_metrics_source                      = "(default-metrics)"
+      default_otlp_metrics_source                 = "default-metrics-otlp"
       kubelet_metrics_source                      = "kubelet-metrics"
       node_metrics_source                         = "node-exporter-metrics"
       scheduler_metrics_source                    = "kube-scheduler-metrics"
@@ -434,6 +435,12 @@ data:
         collector_id = sumologic_collector.collector.id
     }
 
+    resource "sumologic_http_source" "default_otlp_metrics_source" {
+        name         = local.default_otlp_metrics_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "kubelet_metrics_source" {
         name         = local.kubelet_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -473,6 +480,7 @@ data:
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
         endpoint-metrics                          = sumologic_http_source.default_metrics_source.url
+        endpoint-metrics-otlp                     = sumologic_http_source.default_otlp_metrics_source.url
         endpoint-metrics-kubelet                  = sumologic_http_source.kubelet_metrics_source.url
         endpoint-metrics-node-exporter            = sumologic_http_source.node_metrics_source.url
         endpoint-metrics-kube-scheduler           = sumologic_http_source.scheduler_metrics_source.url
@@ -615,6 +623,7 @@ data:
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"
     terraform import sumologic_http_source.default_metrics_source "${COLLECTOR_NAME}/(default-metrics)"
+    terraform import sumologic_http_source.default_otlp_metrics_source "${COLLECTOR_NAME}/default-metrics-otlp"
     terraform import sumologic_http_source.kubelet_metrics_source "${COLLECTOR_NAME}/kubelet-metrics"
     terraform import sumologic_http_source.node_metrics_source "${COLLECTOR_NAME}/node-exporter-metrics"
     terraform import sumologic_http_source.scheduler_metrics_source "${COLLECTOR_NAME}/kube-scheduler-metrics"