feat(logs): add ability to use OTLP source

SumoLogic · May 10, 2023 · 8139771 · 8139771
1 parent 59873c5
commit 8139771
Show file tree

Hide file tree

Showing 25 changed files with 282 additions and 30 deletions.
diff --git a/.changelog/3040.added.txt b/.changelog/3040.added.txt
@@ -0,0 +1 @@
+feat(logs): add ability to use OTLP source
diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md
@@ -89,6 +89,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an
 | `sumologic.logs.defaultFluentd.excludeUnitRegex`                                 | Configuration option only for Fluentd, a regular expression for unit. Matching unit will be excluded from Sumo. The logs will still be sent to FluentD. .                                                                                                                                                                                                            | `""`                                                                                                                                                                                 |
 | `sumologic.logs.fields`                                                          | Fields to be created at Sumo Logic to ensure logs are tagged with relevant metadata. [Sumo Logic help](https://help.sumologic.com/docs/manage/fields/#manage-fields)                                                                                                                                                                                                 | `["cluster", "container", "daemonset", "deployment", "host", "namespace", "node", "pod", "service", "statefulset"]`                                                                  |
 | `sumologic.logs.metadata.provider`                                               | Defines logs metadata enrichment provider - `otelcol` or `fluentd`. `otelcol` is the default and is recommended. `fluentd` is deprecated.                                                                                                                                                                                                                            | `otelcol`                                                                                                                                                                            |
+| `sumologic.logs.sourceType`                                                      | The type of the Sumo Logic source being used for logs ingestion. Can be `http` or `otlp`.                                                                                                                                                                                                                                                                            | `http`                                                                                                                                                                               |
 | `sumologic.metrics.enabled`                                                      | Set the enabled flag to false for disabling metrics ingestion altogether.                                                                                                                                                                                                                                                                                            | `true`                                                                                                                                                                               |
 | `sumologic.metrics.metadata.provider`                                            | Defines metrics metadata enrichment provider - `otelcol` or `fluentd`. `otelcol` is the default and is recommended. `fluentd` is deprecated.                                                                                                                                                                                                                         | `otelcol`                                                                                                                                                                            |
 | `sumologic.metrics.otelcol.extraProcessors`                                      | Extra processors configuration for metrics pipeline. See [/docs/collecting-application-metrics.md#metrics-modifications](/docs/collecting-application-metrics.md#metrics-modifications) for more information.                                                                                                                                                        | `[]`                                                                                                                                                                                 |

diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -1,17 +1,5 @@
-extensions:
-  health_check: {}
-{{ if .Values.metadata.persistence.enabled }}
-  ## Configuration for File Storage extension
-  ## ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/extension/storage/filestorage
-  file_storage:
-    directory: /var/lib/storage/otc
-    timeout: 10s
-    compaction:
-      on_rebound: true
-      directory: /tmp
-{{ end }}
-  pprof: {}
 exporters:
+{{- if eq .Values.sumologic.logs.sourceType "http" }}
 {{ if .Values.sumologic.logs.container.enabled }}
   ## ref: https://github.com/SumoLogic/sumologic-otel-collector/tree/main/pkg/exporter/sumologicexporter
   sumologic/containers:
@@ -26,7 +14,7 @@ exporters:
 {{- if .Values.metadata.persistence.enabled }}
       storage: file_storage
 {{- end }}
-{{ end }}
+{{- end }}
 
 {{ if .Values.sumologic.logs.systemd.enabled }}
   ## ref: https://github.com/SumoLogic/sumologic-otel-collector/tree/main/pkg/exporter/sumologicexporter
@@ -42,7 +30,20 @@ exporters:
 {{- if .Values.metadata.persistence.enabled }}
       storage: file_storage
 {{- end }}
-{{ end }}
+{{- end }}
+
+{{ else }}
+  sumologic:
+    endpoint: ${SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE}
+    log_format: otlp
+    sending_queue:
+      enabled: true
+      num_consumers: 10
+      queue_size: 10_000
+{{- if .Values.metadata.persistence.enabled }}
+      storage: file_storage
+{{- end }}
+{{- end }}
 
 extensions:
 {{ if .Values.metadata.persistence.enabled }}
@@ -509,7 +510,7 @@ service:
 {{ if .Values.sumologic.logs.container.enabled }}
     logs/fluent/containers:
       exporters:
-        - sumologic/containers
+{{ include "logs.otelcol.container.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_containers
@@ -543,7 +544,7 @@ service:
 {{ if .Values.sumologic.logs.systemd.enabled }}
     logs/fluent/kubelet:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.kubelet.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -566,7 +567,7 @@ service:
 
     logs/fluent/systemd:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.systemd.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -598,7 +599,7 @@ service:
     ## - attributes/fluent_containers functionality is being replaced by otellogs operators
     logs/otlp/containers:
       exporters:
-        - sumologic/containers
+{{ include "logs.otelcol.container.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_containers
@@ -634,7 +635,7 @@ service:
     ## - added transform/remove_attributes processor
     logs/otlp/kubelet:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.systemd.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host
@@ -660,7 +661,7 @@ service:
     ## - added transform/remove_attributes processor
     logs/otlp/systemd:
       exporters:
-        - sumologic/systemd
+{{ include "logs.otelcol.systemd.exporters" . | indent 8 }}
       processors:
         - memory_limiter
         - filter/include_fluent_tag_host

diff --git a/deploy/helm/sumologic/templates/_helpers/_logs.tpl b/deploy/helm/sumologic/templates/_helpers/_logs.tpl
@@ -115,6 +115,51 @@ Return the log format for the Sumologic exporter for container logs.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Return the exporters for container log pipeline.
+
+'{{ include "logs.otelcol.container.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.container.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+- sumologic/containers
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the exporters for systemd log pipeline.
+
+'{{ include "logs.otelcol.systemd.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.systemd.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" -}}
+- sumologic/systemd
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else -}}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the exporters for kubelet log pipeline.
+
+'{{ include "logs.otelcol.kubelet.exporters" . }}'
+*/}}
+{{- define "logs.otelcol.kubelet.exporters" -}}
+{{- if eq .Values.sumologic.logs.sourceType "http" }}
+- sumologic/systemd
+{{- else if eq .Values.sumologic.logs.sourceType "otlp" }}
+- sumologic
+{{- else }}
+{{- fail "`sumologic.logs.sourceType` can only be `http` or `otlp`" -}}
+{{- end -}}
+{{- end -}}
+
 {{- define "sumologic.labels.app.logs" -}}
 {{- if eq .Values.sumologic.logs.metadata.provider "fluentd" -}}
 {{ template "sumologic.labels.app.fluentd" . }}-logs

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -199,6 +199,11 @@ sumologic:
           #   - name: "Test Exclude Debug"
           #     filter_type: "Exclude"
           #     regexp: ".*DEBUG.*"
+        default-otlp:
+          name: logs-otlp
+          config-name: endpoint-logs-otlp
+          properties:
+            content_type: Otlp
       events:
         default:
           name: events
@@ -406,6 +411,8 @@ sumologic:
       - service
       - statefulset
 
+    sourceType: http
+
   ### Metrics configuration
   ## Set the enabled flag to false for disabling metrics ingestion altogether.
   metrics:

diff --git a/docs/otlp-source.md b/docs/otlp-source.md
@@ -0,0 +1,41 @@
+# Using the OTLP source
+
+Historically, agents used by this Chart sent logs and metrics data to a [generic HTTP Source][http_source] in Sumo Logic. Ever since the
+[version 3][v3] release, the Chart predominantly uses the [OpenTelemetry Collector][otel], and consequently also the [OTLP protocol][otlp].
+The data needed to be converted to the formats the generic HTTP source supports before being sent.
+
+Recently, we've added support for directly sending data using the OTLP protocol to Sumo Logic. This is achieved using the [OTLP
+source][otlp_source]. This document explains how to use this new source with the Helm Chart.
+
+**Note:** OTLP Source is currently available for the following signal types: logs
+
+## Enabling the OTLP source
+
+Add the following to your configuration:
+
+```yaml
+sumologic:
+  logs:
+    sourceType: otlp
+```
+
+**Note:** The source is automatically created during Chart installation. This setting simply makes the Chart start sending data to it.
+
+## Benefits
+
+Sending data directly via OTLP is more efficient, as we skip the conversion step. OTLP is also a binary-encoded format, which improves the
+efficiency further.
+
+### Logs
+
+As a structured log format, OTLP frees us from the need to parse metadata out of the log body on the Sumo side. This makes the following
+features work without additional manual configuration:
+
+- multiline parsing for the `text` log format
+- correct timestamps for the `text` log format
+
+[http_source]: https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/logs-metrics/
+[otlp_source]: https://help.sumologic.com/docs/send-data/hosted-collectors/http-source/otlp/
+[v3]: https://github.com/SumoLogic/sumologic-kubernetes-collection/releases/tag/v3.0.0
+[otel]: ./opentelemetry-collector.md
+[otlp]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md
diff --git a/tests/helm/logs_test.go b/tests/helm/logs_test.go
@@ -259,6 +259,37 @@ sumologic:
 	}
 }
 
+func TestMetadataLogOtlpSource(t *testing.T) {
+	t.Parallel()
+	templatePath := "templates/logs/otelcol/configmap.yaml"
+	valuesYaml := `
+sumologic:
+  logs:
+    sourceType: otlp
+`
+
+	var otelConfig struct {
+		Exporters  map[string]map[string]interface{}
+		Processors map[string]interface{}
+		Service    struct {
+			Pipelines map[string]struct {
+				Receivers  []string
+				Processors []string
+				Exporters  []string
+			}
+		}
+	}
+
+	otelConfigYaml := GetOtelConfigYaml(t, valuesYaml, templatePath)
+	err := yaml.Unmarshal([]byte(otelConfigYaml), &otelConfig)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []string{"sumologic"}, keys(otelConfig.Exporters))
+	require.Equal(t, "otlp", otelConfig.Exporters["sumologic"]["log_format"])
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/containers"].Exporters)
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/systemd"].Exporters)
+	require.ElementsMatch(t, []string{"sumologic"}, otelConfig.Service.Pipelines["logs/otlp/kubelet"].Exporters)
+}
+
 func TestCollectorOtelConfigMerge(t *testing.T) {
 	t.Parallel()
 	templatePath := "templates/logs/collector/otelcol/configmap.yaml"

diff --git a/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml b/tests/helm/testdata/goldenfile/metadata_logs_otc_statefulset/basic.output.yaml
@@ -91,11 +91,11 @@ spec:
             - name: pprof
               containerPort: 1777
               protocol: TCP
-            - name: otlphttp2
-              containerPort: 4319
+            - containerPort: 4319
+              name: otlphttp2
               protocol: TCP
-            - name: otlphttp3
-              containerPort: 4320
+            - containerPort: 4320
+              name: otlphttp3
               protocol: TCP
           livenessProbe:
             httpGet:
@@ -136,6 +136,12 @@ spec:
                 secretKeyRef:
                   name: sumologic
                   key: endpoint-logs
+            - name: SUMO_ENDPOINT_DEFAULT_OTLP_LOGS_SOURCE
+              valueFrom:
+                secretKeyRef:
+                  name: sumologic
+                  key: endpoint-logs-otlp
+
             - name: NO_PROXY
               value: kubernetes.default.svc
             - name: VALUE_FROM_SECRET

diff --git a/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/all_fields.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -483,6 +484,12 @@ data:
         }
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -583,6 +590,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -726,6 +734,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"

diff --git a/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml b/tests/helm/testdata/goldenfile/terraform/collector_fields.output.yaml
@@ -344,6 +344,7 @@ data:
     locals {
       default_events_source                       = "events"
       default_logs_source                         = "logs"
+      default_otlp_logs_source                    = "logs-otlp"
       apiserver_metrics_source                    = "apiserver-metrics"
       control_plane_metrics_source                = "control-plane-metrics"
       controller_metrics_source                   = "kube-controller-manager-metrics"
@@ -484,6 +485,12 @@ data:
         }
     }
 
+    resource "sumologic_http_source" "default_otlp_logs_source" {
+        name         = local.default_otlp_logs_source
+        collector_id = sumologic_collector.collector.id
+        content_type = "Otlp"
+    }
+
     resource "sumologic_http_source" "apiserver_metrics_source" {
         name         = local.apiserver_metrics_source
         collector_id = sumologic_collector.collector.id
@@ -539,6 +546,7 @@ data:
       data = {
         endpoint-events                           = sumologic_http_source.default_events_source.url
         endpoint-logs                             = sumologic_http_source.default_logs_source.url
+        endpoint-logs-otlp                        = sumologic_http_source.default_otlp_logs_source.url
         endpoint-metrics-apiserver                = sumologic_http_source.apiserver_metrics_source.url
         endpoint-control_plane_metrics_source     = sumologic_http_source.control_plane_metrics_source.url
         endpoint-metrics-kube-controller-manager  = sumologic_http_source.controller_metrics_source.url
@@ -681,6 +689,7 @@ data:
     true  # prevent to render empty if; then
     terraform import sumologic_http_source.default_events_source "${COLLECTOR_NAME}/events"
     terraform import sumologic_http_source.default_logs_source "${COLLECTOR_NAME}/logs"
+    terraform import sumologic_http_source.default_otlp_logs_source "${COLLECTOR_NAME}/logs-otlp"
     terraform import sumologic_http_source.apiserver_metrics_source "${COLLECTOR_NAME}/apiserver-metrics"
     terraform import sumologic_http_source.control_plane_metrics_source "${COLLECTOR_NAME}/control-plane-metrics"
     terraform import sumologic_http_source.controller_metrics_source "${COLLECTOR_NAME}/kube-controller-manager-metrics"