feat(metrics): remove app metrics filtering from metrics collector

.changelog/3232.changed.txt

@@ -0,0 +1 @@
+feat(metrics): move app metrics filtering to metadata layer

deploy/helm/sumologic/conf/metrics/collector/otelcol/config.yaml

@@ -61,10 +61,6 @@ processors:
         - IsMatch(name, "^(apiserver_request_duration_seconds|coredns_dns_request_duration_seconds|kubelet_runtime_operations_duration_seconds)$")
 {{- end }}
 
-{{ if .Values.sumologic.metrics.enableDefaultFilters -}}
-{{ tpl (.Files.Get "conf/metrics/collector/otelcol/default-filters.yaml") . | indent 2 }}
-{{ end -}}
-
 receivers:
   prometheus:
     config:
@@ -222,8 +218,5 @@ service:
 {{- end }}
         - filter/drop_unnecessary_metrics
         - transform/drop_unnecessary_attributes
-{{- if .Values.sumologic.metrics.enableDefaultFilters }}
-        - filter/app_metrics
-{{- end }}
       receivers: [prometheus]
 

deploy/helm/sumologic/values.yaml

@@ -1197,6 +1197,10 @@ kube-prometheus-stack:
               target_label: _sumo_forward_
               replacement: "true"
               action: replace
+      remoteWrite:
+        ## infrastructure metrics
+        - url: http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888/prometheus.metrics
+          remoteTimeout: 5s
 
     serviceMonitor:
       selfMonitor: false

tests/helm/testdata/goldenfile/metadata_metrics_otc/additional_endpoints.output.yaml

@@ -281,6 +281,7 @@ data:
             data_format = "prometheusremotewrite"
             path_tag = true
             paths = [
+              "/prometheus.metrics",
               "/prometheus.metrics.custom",
               "/prometheus.metrics.others"
             ]

tests/helm/testdata/goldenfile/metadata_metrics_otc/basic.output.yaml

@@ -268,10 +268,21 @@ data:
           http:
             endpoint: 0.0.0.0:4318
       telegraf:
-        agent_config: "[agent]\n  interval = \"30s\"\n  flush_interval = \"30s\"\n  omit_hostname
-          = true\n[[inputs.http_listener_v2]]\n  # wait longer than prometheus\n  read_timeout
-          = \"30s\"\n  write_timeout = \"30s\"\n  service_address = \":9888\"\n  data_format
-          = \"prometheusremotewrite\"\n  path_tag = true\n  paths = [\n    \n  ]\n"
+        agent_config: |
+          [agent]
+            interval = "30s"
+            flush_interval = "30s"
+            omit_hostname = true
+          [[inputs.http_listener_v2]]
+            # wait longer than prometheus
+            read_timeout = "30s"
+            write_timeout = "30s"
+            service_address = ":9888"
+            data_format = "prometheusremotewrite"
+            path_tag = true
+            paths = [
+              "/prometheus.metrics"
+            ]
     service:
       extensions:
       - health_check

tests/helm/testdata/goldenfile/metadata_metrics_otc/custom.output.yaml

@@ -149,10 +149,21 @@ data:
           http:
             endpoint: 0.0.0.0:4318
       telegraf:
-        agent_config: "[agent]\n  interval = \"30s\"\n  flush_interval = \"30s\"\n  omit_hostname
-          = true\n[[inputs.http_listener_v2]]\n  # wait longer than prometheus\n  read_timeout
-          = \"30s\"\n  write_timeout = \"30s\"\n  service_address = \":9888\"\n  data_format
-          = \"prometheusremotewrite\"\n  path_tag = true\n  paths = [\n    \n  ]\n"
+        agent_config: |
+          [agent]
+            interval = "30s"
+            flush_interval = "30s"
+            omit_hostname = true
+          [[inputs.http_listener_v2]]
+            # wait longer than prometheus
+            read_timeout = "30s"
+            write_timeout = "30s"
+            service_address = ":9888"
+            data_format = "prometheusremotewrite"
+            path_tag = true
+            paths = [
+              "/prometheus.metrics"
+            ]
     service:
       extensions:
       - health_check

tests/helm/testdata/goldenfile/metadata_metrics_otc/filtered_app_metrics.output.yaml

@@ -292,10 +292,21 @@ data:
           http:
             endpoint: 0.0.0.0:4318
       telegraf:
-        agent_config: "[agent]\n  interval = \"30s\"\n  flush_interval = \"30s\"\n  omit_hostname
-          = true\n[[inputs.http_listener_v2]]\n  # wait longer than prometheus\n  read_timeout
-          = \"30s\"\n  write_timeout = \"30s\"\n  service_address = \":9888\"\n  data_format
-          = \"prometheusremotewrite\"\n  path_tag = true\n  paths = [\n    \n  ]\n"
+        agent_config: |
+          [agent]
+            interval = "30s"
+            flush_interval = "30s"
+            omit_hostname = true
+          [[inputs.http_listener_v2]]
+            # wait longer than prometheus
+            read_timeout = "30s"
+            write_timeout = "30s"
+            service_address = ":9888"
+            data_format = "prometheusremotewrite"
+            path_tag = true
+            paths = [
+              "/prometheus.metrics"
+            ]
     service:
       extensions:
       - health_check