fix(logs/metadata): fix logs metadata for systemd

CHANGELOG.md

@@ -24,10 +24,12 @@ with the [old configuration][source_processor_old_config] and apply correspondin
 
 - chore(traces): switch OTC fork to OTel Distro [#2334][#2334]
 - chore: add support for Kops 1.23 [#2361][#2361]
+- fix(logs/metadata): fix logs metadata for systemd [#2363]
 
 [#2334]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2334
 [#2172]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2172
 [#2361]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2361
+[#2363]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2363
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.9.1...main
 
 ## [v2.9.1]

deploy/helm/sumologic/values.yaml

@@ -4166,6 +4166,13 @@ metadata:
               record_attributes:
                 - key: fluent.tag
                   value: containers\..+
+        filter/include_containers:
+          logs:
+            include:
+              match_type: regexp
+              record_attributes:
+                - key: k8s.container.name
+                  value: .+
         attributes/containers:
           actions:
             - action: extract
@@ -4271,11 +4278,14 @@ metadata:
               record_attributes:
                 - key: fluent.tag
                   value: host\..+
-        attributes/extract_systemd_source_name_from_fluent_tag:
+        attributes/extract_systemd_source_fields:
           actions:
             - action: extract
               key: fluent.tag
               pattern: ^host\.(?P<_sourceName>[a-zA-z0-9]+)\..+$
+            - action: insert
+              from_attribute: _HOSTNAME
+              key: _sourceHost
         filter/include_systemd:
           logs:
             include:
@@ -4350,11 +4360,12 @@ metadata:
         groupbyattrs/systemd:
           keys:
             - _sourceName
+            - _sourceHost
             - _collector
         source/systemd:
           collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
-          source_host: "%{_HOSTNAME}"
-          source_name: '%{_sourceName}'
+          source_host: "%{_sourceHost}"
+          source_name: "%{_sourceName}"
           source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}'
           source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}'
           source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}'
@@ -4369,7 +4380,7 @@ metadata:
                   value: kubelet.service
         source/kubelet:
           collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}'
-          source_host: "%{_HOSTNAME}"
+          source_host: "%{_sourceHost}"
           source_name: '{{ .Values.fluentd.logs.kubelet.sourceName | quote }}'
           source_category: '{{ .Values.fluentd.logs.kubelet.sourceCategory | quote }}'
           source_category_prefix: '{{ .Values.fluentd.logs.kubelet.sourceCategoryPrefix | quote }}'
@@ -4412,6 +4423,7 @@ metadata:
           #     - otlp
           #   processors:
           #     - memory_limiter
+          #     - filter/include_containers
           #     - groupbyattrs/containers
           #     - k8s_tagger
           #     - resource/add_cluster
@@ -4433,10 +4445,11 @@ metadata:
               - filter/exclude_systemd_hostname
               - filter/exclude_systemd_priority
               - filter/exclude_systemd_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
               - groupbyattrs/systemd
               - resource/add_cluster
               - source/systemd
-              - attributes/remove_fluent_tag
               - batch
             exporters:
               - sumologic/systemd
@@ -4451,10 +4464,11 @@ metadata:
               - filter/exclude_kubelet_hostname
               - filter/exclude_kubelet_priority
               - filter/exclude_kubelet_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
               - groupbyattrs/systemd
               - resource/add_cluster
               - source/kubelet
-              - attributes/remove_fluent_tag
               - batch
             exporters:
               - sumologic/systemd

tests/helm/metadata_logs_otc/static/basic.output.yaml

@@ -69,11 +69,14 @@ data:
           key: k8s.container.name
         - action: delete
           key: k8s_container_name
-      attributes/extract_systemd_source_name_from_fluent_tag:
+      attributes/extract_systemd_source_fields:
         actions:
         - action: extract
           key: fluent.tag
           pattern: ^host\.(?P<_sourceName>[a-zA-z0-9]+)\..+$
+        - action: insert
+          from_attribute: _HOSTNAME
+          key: _sourceHost
       attributes/remove_fluent_tag:
         actions:
         - action: delete
@@ -144,6 +147,13 @@ data:
             record_attributes:
             - key: _SYSTEMD_UNIT
               value: "$^"
+      filter/include_containers:
+        logs:
+          include:
+            match_type: regexp
+            record_attributes:
+            - key: k8s.container.name
+              value: .+
       filter/include_fluent_tag_containers:
         logs:
           include:
@@ -182,6 +192,7 @@ data:
       groupbyattrs/systemd:
         keys:
         - _sourceName
+        - _sourceHost
         - _collector
       k8s_tagger:
         extract:
@@ -254,14 +265,14 @@ data:
         source_category: "kubelet"
         source_category_prefix: "kubernetes/"
         source_category_replace_dash: "/"
-        source_host: '%{_HOSTNAME}'
+        source_host: '%{_sourceHost}'
         source_name: "k8s_kubelet"
       source/systemd:
         collector: "kubernetes"
         source_category: "system"
         source_category_prefix: "kubernetes/"
         source_category_replace_dash: "/"
-        source_host: '%{_HOSTNAME}'
+        source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
     receivers:
       fluentforward:
@@ -304,10 +315,11 @@ data:
           - filter/exclude_kubelet_hostname
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
+          - attributes/extract_systemd_source_fields
+          - attributes/remove_fluent_tag
           - groupbyattrs/systemd
           - resource/add_cluster
           - source/kubelet
-          - attributes/remove_fluent_tag
           - batch
           receivers:
           - fluentforward
@@ -323,10 +335,11 @@ data:
           - filter/exclude_systemd_hostname
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
+          - attributes/extract_systemd_source_fields
+          - attributes/remove_fluent_tag
           - groupbyattrs/systemd
           - resource/add_cluster
           - source/systemd
-          - attributes/remove_fluent_tag
           - batch
           receivers:
           - fluentforward

tests/helm/metadata_logs_otc/static/templates.output.yaml

@@ -69,11 +69,14 @@ data:
           key: k8s.container.name
         - action: delete
           key: k8s_container_name
-      attributes/extract_systemd_source_name_from_fluent_tag:
+      attributes/extract_systemd_source_fields:
         actions:
         - action: extract
           key: fluent.tag
           pattern: ^host\.(?P<_sourceName>[a-zA-z0-9]+)\..+$
+        - action: insert
+          from_attribute: _HOSTNAME
+          key: _sourceHost
       attributes/remove_fluent_tag:
         actions:
         - action: delete
@@ -144,6 +147,13 @@ data:
             record_attributes:
             - key: _SYSTEMD_UNIT
               value: "my_systemd_excludeUnitRegex"
+      filter/include_containers:
+        logs:
+          include:
+            match_type: regexp
+            record_attributes:
+            - key: k8s.container.name
+              value: .+
       filter/include_fluent_tag_containers:
         logs:
           include:
@@ -182,6 +192,7 @@ data:
       groupbyattrs/systemd:
         keys:
         - _sourceName
+        - _sourceHost
         - _collector
       k8s_tagger:
         extract:
@@ -254,14 +265,14 @@ data:
         source_category: "kubelet"
         source_category_prefix: "my_kubelet_sourceCategoryPrefix"
         source_category_replace_dash: "my_kubelet_sourceCategoryReplaceDash"
-        source_host: '%{_HOSTNAME}'
+        source_host: '%{_sourceHost}'
         source_name: "k8s_kubelet"
       source/systemd:
         collector: "my_collectorName"
         source_category: "system"
         source_category_prefix: "my_systemd_sourceCategoryPrefix"
         source_category_replace_dash: "my_systemd_sourceCategoryReplaceDash"
-        source_host: '%{_HOSTNAME}'
+        source_host: '%{_sourceHost}'
         source_name: '%{_sourceName}'
     receivers:
       fluentforward:
@@ -304,10 +315,11 @@ data:
           - filter/exclude_kubelet_hostname
           - filter/exclude_kubelet_priority
           - filter/exclude_kubelet_unit
+          - attributes/extract_systemd_source_fields
+          - attributes/remove_fluent_tag
           - groupbyattrs/systemd
           - resource/add_cluster
           - source/kubelet
-          - attributes/remove_fluent_tag
           - batch
           receivers:
           - fluentforward
@@ -323,10 +335,11 @@ data:
           - filter/exclude_systemd_hostname
           - filter/exclude_systemd_priority
           - filter/exclude_systemd_unit
+          - attributes/extract_systemd_source_fields
+          - attributes/remove_fluent_tag
           - groupbyattrs/systemd
           - resource/add_cluster
           - source/systemd
-          - attributes/remove_fluent_tag
           - batch
           receivers:
           - fluentforward

vagrant/values.yaml

@@ -249,38 +249,34 @@ metadata:
             exporters:
               - sumologic/containers
           logs/fluent/systemd:
-            receivers:
-              - fluentforward
             processors:
               - memory_limiter
               - filter/include_fluent_tag_host
-
-              # Vagrant specific
-              - filter/exclude_fluent_tag_receiver_mock_container
-
               - filter/include_systemd
               - filter/exclude_systemd_snap_kubelite
-              - attributes/extract_systemd_source_name_from_fluent_tag
+              - filter/exclude_kubelet
+              - filter/exclude_systemd_syslog
+              - filter/exclude_systemd_hostname
+              - filter/exclude_systemd_priority
+              - filter/exclude_systemd_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/systemd
-              - attributes/remove_fluent_tag
               - batch
-            exporters:
-              - sumologic/systemd
           logs/fluent/kubelet:
-            receivers:
-              - fluentforward
             processors:
               - memory_limiter
               - filter/include_fluent_tag_host
-
-              # Vagrant specific
-              - filter/exclude_fluent_tag_receiver_mock_container
-
               - filter/include_systemd_snap_kubelite
+              - filter/exclude_kubelet_syslog
+              - filter/exclude_kubelet_hostname
+              - filter/exclude_kubelet_priority
+              - filter/exclude_kubelet_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
               - groupbyattrs/systemd
+              - resource/add_cluster
               - source/kubelet
-              - attributes/remove_fluent_tag
               - batch
-            exporters:
-              - sumologic/systemd