otelcol: add systemd pipeline

SumoLogic · Sep 20, 2021 · a096eef · a096eef
1 parent 2493ad5
commit a096eef
Show file tree

Hide file tree

Showing 4 changed files with 145 additions and 19 deletions.
diff --git a/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml b/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml
@@ -125,7 +125,7 @@ spec:
             path: /fluentd.pod.healthcheck?json=%7B%22log%22%3A+%22health+check%22%7D
             port: 9880
           initialDelaySeconds: 300
-          periodSeconds: 30
+          periodSeconds: 5
           timeoutSeconds: 3
         readinessProbe:
           httpGet:

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -2769,7 +2769,7 @@ otelcol:
         extensions:
           health_check: {}
         exporters:
-          sumologic:
+          sumologic/containers:
             log_format: json
             endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
             ## ToDo: Move sources to sourceprocessor
@@ -2789,14 +2789,27 @@ otelcol:
               - host
               - node
               - pod
+          sumologic/systemd:
+            log_format: json
+            endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
+            ## ToDo: Move sources to sourceprocessor
+            source_name: "%{_sourceName}"
+            source_category: "%{_sourceCategory}"
+            source_host: "%{_sourceHost}"
+            sending_queue:
+              enabled: true
+            metadata_attributes:
+              - _collector
+              - _sourceCategory
+              - _sourceHost
+              - _sourceName
+
         processors:
-          attributes:
+          attributes/containers:
             actions:
               - action: extract
                 key: fluent.tag
                 pattern: ^containers\.var\.log\.containers\.(?P<k8s_pod_name>[^_]+)_(?P<k8s_namespace>[^_]+)_(?P<k8s_container_name>.+)-(?P<container_id>[a-f0-9]{64})\.log$
-              - action: delete
-                key: fluent.tag
 
               - action: insert
                 key: k8s.container.id
@@ -2821,12 +2834,46 @@ otelcol:
                 from_attribute: k8s_container_name
               - action: delete
                 key: k8s_container_name
-          groupbyattrs:
+          attributes/remove_fluent_tag:
+            actions:
+              - action: delete
+                key: fluent.tag
+
+          groupbyattrs/fluent_tag:
+            keys:
+              - fluent.tag
+          groupbyattrs/containers:
             keys:
               - k8s.container.id
               - k8s.container.name
               - k8s.namespace.name
               - k8s.pod.name
+          groupbyattrs/systemd:
+            keys:
+              - _SYSTEMD_UNIT
+              - _HOSTNAME
+
+          filter/include_fluent_tag_containers:
+            logs:
+              include:
+                match_type: regexp
+                record_attributes:
+                  - key: fluent.tag
+                    value: containers\..+
+          filter/include_fluent_tag_host:
+            logs:
+              include:
+                match_type: regexp
+                record_attributes:
+                  - key: fluent.tag
+                    value: host\..+
+          filter/include_systemd:
+            logs:
+              include:
+                match_type: regexp
+                record_attributes:
+                  - key: _SYSTEMD_UNIT
+                    value: .+
 
           ## The memory_limiter processor is used to prevent out of memory situations on the collector.
           memory_limiter:
@@ -2879,7 +2926,7 @@ otelcol:
               delimiter: "_"
             pod_association:
               - from: build_hostname
-          source:
+          source/containers:
             collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
             source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}'
             source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}'
@@ -2897,6 +2944,14 @@ otelcol:
             pod_key: "k8s.pod.name"
             container_key: "k8s.container.name"
             source_host_key: "k8s.pod.hostname"
+          source/systemd:
+            collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
+            source_name: '{{ .Values.fluentd.logs.systemd.sourceName | quote }}'
+            source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}'
+            source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}'
+            source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}'
+            source_host_key: "_HOSTNAME"
+
         service:
           extensions:
             - health_check
@@ -2907,13 +2962,28 @@ otelcol:
                 - fluentforward
               processors:
                 - memory_limiter
-                - attributes
-                - groupbyattrs
+                - filter/include_fluent_tag_containers
+                - attributes/containers
+                - attributes/remove_fluent_tag
+                - groupbyattrs/containers
                 - k8s_tagger
-                - source
+                - source/containers
                 - batch
               exporters:
-                - sumologic
+                - sumologic/containers
+            logs/systemd:
+              receivers:
+                - fluentforward
+              processors:
+                - memory_limiter
+                - filter/include_fluent_tag_host
+                - attributes/remove_fluent_tag
+                - filter/include_systemd
+                - groupbyattrs/systemd
+                - source/systemd
+                - batch
+              exporters:
+                - sumologic/systemd
   deployment:
     nodeSelector: {}
     tolerations: []

diff --git a/vagrant/k8s/receiver-mock.yaml b/vagrant/k8s/receiver-mock.yaml
@@ -35,6 +35,8 @@ spec:
           args:
             - receiver-mock
             - --hostname=receiver-mock.receiver-mock
+            - --print-headers
+            - --print-logs
           resources: {}
           securityContext:
             capabilities:

diff --git a/vagrant/values.yaml b/vagrant/values.yaml
@@ -7,14 +7,14 @@ kube-prometheus-stack:
       dashboardproviders.yaml:
         apiVersion: 1
         providers:
-        - name: 'default'
-          orgId: 1
-          folder: ''
-          type: file
-          disableDeletion: false
-          editable: true
-          options:
-            path: /var/lib/grafana/dashboards/default
+          - name: 'default'
+            orgId: 1
+            folder: ''
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/default
     sidecar:
       image:
         sha: ''
@@ -148,3 +148,57 @@ fluentd:
         </match>
       multiline:
         enabled: false
+
+otelcol:
+  metadata:
+    logs:
+      config:
+        processors:
+          # Filter out receiver-mock logs to prevent snowball effect
+          filter/fluent_tag_receiver_mock_container:
+            logs:
+              exclude:
+                match_type: regexp
+                record_attributes:
+                  - key: fluent.tag
+                    value: containers\.var\.log\.containers\.receiver-mock.*
+        service:
+          extensions:
+            - health_check
+            # - sumologic
+          pipelines:
+            logs:
+              receivers:
+                - fluentforward
+              processors:
+                - memory_limiter
+                - filter/include_fluent_tag_containers
+
+                # Vagrant specific
+                - filter/fluent_tag_receiver_mock_container
+
+                - attributes/containers
+                - attributes/remove_fluent_tag
+                - groupbyattrs/containers
+                - k8s_tagger
+                - source/containers
+                - batch
+              exporters:
+                - sumologic/containers
+            logs/systemd:
+              receivers:
+                - fluentforward
+              processors:
+                - memory_limiter
+                - filter/include_fluent_tag_host
+
+                # Vagrant specific
+                - filter/fluent_tag_receiver_mock_container
+
+                - attributes/remove_fluent_tag
+                - filter/include_systemd
+                - groupbyattrs/systemd
+                - source/systemd
+                - batch
+              exporters:
+                - sumologic/systemd