feat(helm): move relabelling in prometheus from remoteWrites to servi…

…ceMonitors (#3103) * feat(helm): move relabelling in prometheus from remoteWrites to serviceMonitors Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * Update deploy/helm/sumologic/values.yaml --------- Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
SumoLogic · Jun 28, 2023 · e437b13 · e437b13
1 parent 5f358b1
commit e437b13
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 15 deletions.
diff --git a/.changelog/3103.changed.txt b/.changelog/3103.changed.txt
@@ -0,0 +1 @@
+feat(helm): move relabelling in prometheus from remoteWrites to serviceMonitors
diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -1885,6 +1885,23 @@ kube-prometheus-stack:
         - action: keep
           regex: (?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_hpa_spec_max_replicas|kube_hpa_spec_min_replicas|kube_hpa_status_(condition|(current|desired)_replicas)|kube_pod_container_info|kube_pod_container_resource_requests|kube_pod_container_resource_limits|kube_pod_container_status_ready|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting_reason|kube_pod_container_status_restarts_total|kube_pod_status_phase|kube_pod_info|kube_service_info|kube_service_spec_external_ip|kube_service_spec_type|kube_service_status_load_balancer_ingress)
           sourceLabels: [__name__]
+        ## Drop unnecessary labels Prometheus adds to these metrics
+        ## We don't want container=kube-state-metrics on everything
+        - action: labeldrop
+          regex: service
+        - action: replace
+          sourceLabels: [container]
+          regex: kube-state-metrics
+          targetLabel: container
+          replacement: ""
+        - action: replace
+          sourceLabels: [pod]
+          regex: ".*kube-state-metrics.*"
+          targetLabel: pod
+          replacement: ""
+        - action: labelmap
+          regex: (pod|service)
+          replacement: service_discovery_${1}
   nodeExporter:
     serviceMonitor:
       ## Scrape interval. If not set, the Prometheus default scrape interval is used.
@@ -2140,11 +2157,6 @@ kube-prometheus-stack:
             - action: keep
               regex: kube-state-metrics;(?:kube_statefulset_status_observed_generation|kube_statefulset_status_replicas|kube_statefulset_replicas|kube_statefulset_metadata_generation|kube_daemonset_status_current_number_scheduled|kube_daemonset_status_desired_number_scheduled|kube_daemonset_status_number_misscheduled|kube_daemonset_status_number_unavailable|kube_deployment_spec_replicas|kube_deployment_status_replicas_available|kube_deployment_status_replicas_unavailable|kube_node_info|kube_node_status_allocatable|kube_node_status_capacity|kube_node_status_condition|kube_hpa_spec_max_replicas|kube_hpa_spec_min_replicas|kube_hpa_status_(condition|(current|desired)_replicas)|kube_service_info|kube_service_spec_external_ip|kube_service_spec_type|kube_service_status_load_balancer_ingress)
               sourceLabels: [job, __name__]
-            - action: labelmap
-              regex: (pod|service)
-              replacement: service_discovery_${1}
-            - action: labeldrop
-              regex: (pod|service|container)
         ## kube pod state metrics
         ## kube_pod_status_phase
         - url: http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888/prometheus.metrics.state
@@ -2153,8 +2165,6 @@ kube-prometheus-stack:
             - action: keep
               regex: kube-state-metrics;(?:kube_pod_status_phase)
               sourceLabels: [job, __name__]
-            - action: labeldrop
-              regex: container
         ## kube container state metrics
         ## kube_pod_container_info
         ## kube_pod_container_resource_limits
@@ -2259,12 +2269,6 @@ kube-prometheus-stack:
         - url: http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888/prometheus.metrics.container
           remoteTimeout: 5s
           writeRelabelConfigs:
-            - action: labelmap
-              regex: container_name
-              replacement: container
-            - action: drop
-              regex: POD
-              sourceLabels: [container]
             - action: keep
               regex: kubelet;.+;(?:container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_fs_usage_bytes|container_fs_limit_bytes|container_cpu_cfs_throttled_seconds_total)
               sourceLabels: [job, container, __name__]
@@ -2277,8 +2281,6 @@ kube-prometheus-stack:
             - action: keep
               regex: kubelet;(?:container_network_receive_bytes_total|container_network_transmit_bytes_total)
               sourceLabels: [job, __name__]
-            - action: labeldrop
-              regex: container
         ## node exporter metrics
         ## node_cpu_seconds_total
         ## node_load1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		feat(helm): move relabelling in prometheus from remoteWrites to serviceMonitors