diff --git a/Makefile b/Makefile index 0c12f2e06..a1bc50615 100644 --- a/Makefile +++ b/Makefile @@ -146,6 +146,9 @@ define deploy kubectl get pods $(BIN)/helm upgrade --install $(1) ./operations/phlare/helm/phlare $(2) \ --set phlare.image.tag=$(IMAGE_TAG) --set phlare.image.repository=$(IMAGE_PREFIX)phlare --set phlare.service.port_name=http-metrics \ + --set phlare.podAnnotations."profiles\.grafana\.com\/memory\.port_name"=http-metrics \ + --set phlare.podAnnotations."profiles\.grafana\.com\/cpu\.port_name"=http-metrics \ + --set phlare.podAnnotations."profiles\.grafana\.com\/goroutine\.port_name"=http-metrics \ --set phlare.components.querier.resources=null --set phlare.components.distributor.resources=null --set phlare.components.ingester.resources=null endef diff --git a/docs/sources/operators-guide/deploy-kubernetes/_index.md b/docs/sources/operators-guide/deploy-kubernetes/_index.md index 0199f5c09..63817c889 100644 --- a/docs/sources/operators-guide/deploy-kubernetes/_index.md +++ b/docs/sources/operators-guide/deploy-kubernetes/_index.md @@ -171,9 +171,9 @@ Modify the Helm deployment by running: ## Optional: Scrape your own workload's profiles The Phlare chart uses a default configuration that causes its agent to scrape Pods, provided they have the correct annotations. -This functionality uses [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) and [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) that might be familar from the Prometheus or Grafna Agent config. +This functionality uses [relabel_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config) and [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) you might be familiar with the Prometheus or Grafana Agent config. -In order to get Phlare to scrape pods, you must add the following annotations to the the pods: +To get Phlare to scrape pods, you must add the following annotations to the pods: ```yaml metadata: diff --git a/operations/phlare/helm/phlare/Chart.yaml b/operations/phlare/helm/phlare/Chart.yaml index fd8f22e7b..64a017427 100644 --- a/operations/phlare/helm/phlare/Chart.yaml +++ b/operations/phlare/helm/phlare/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: phlare description: 🔥 horizontally-scalable, highly-available, multi-tenant continuous profiling aggregation system type: application -version: 0.5.2 +version: 0.5.3 appVersion: 0.5.1 dependencies: - name: minio diff --git a/operations/phlare/helm/phlare/README.md b/operations/phlare/helm/phlare/README.md index 8fd690043..316f04a54 100644 --- a/operations/phlare/helm/phlare/README.md +++ b/operations/phlare/helm/phlare/README.md @@ -1,6 +1,6 @@ # phlare -![Version: 0.5.2](https://img.shields.io/badge/Version-0.5.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.5.1](https://img.shields.io/badge/AppVersion-0.5.1-informational?style=flat-square) +![Version: 0.5.3](https://img.shields.io/badge/Version-0.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.5.1](https://img.shields.io/badge/AppVersion-0.5.1-informational?style=flat-square) 🔥 horizontally-scalable, highly-available, multi-tenant continuous profiling aggregation system @@ -38,8 +38,11 @@ | phlare.persistence.annotations | object | `{}` | | | phlare.persistence.enabled | bool | `false` | | | phlare.persistence.size | string | `"10Gi"` | | +| phlare.podAnnotations."profiles.grafana.com/cpu.port_name" | string | `"http2"` | | | phlare.podAnnotations."profiles.grafana.com/cpu.scrape" | string | `"true"` | | +| phlare.podAnnotations."profiles.grafana.com/goroutine.port_name" | string | `"http2"` | | | phlare.podAnnotations."profiles.grafana.com/goroutine.scrape" | string | `"true"` | | +| phlare.podAnnotations."profiles.grafana.com/memory.port_name" | string | `"http2"` | | | phlare.podAnnotations."profiles.grafana.com/memory.scrape" | string | `"true"` | | | phlare.podSecurityContext.fsGroup | int | `10001` | | | phlare.podSecurityContext.runAsNonRoot | bool | `true` | | diff --git a/operations/phlare/helm/phlare/rendered/micro-services.yaml b/operations/phlare/helm/phlare/rendered/micro-services.yaml index 6b8ee7f3b..f8c404c86 100644 --- a/operations/phlare/helm/phlare/rendered/micro-services.yaml +++ b/operations/phlare/helm/phlare/rendered/micro-services.yaml @@ -12,7 +12,7 @@ kind: ServiceAccount metadata: name: phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -358,7 +358,7 @@ kind: ConfigMap metadata: name: phlare-dev-overrides-config labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -374,7 +374,7 @@ kind: ConfigMap metadata: name: phlare-dev-config labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -406,10 +406,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -527,6 +523,9 @@ data: enabled: false mutex: enabled: false + process_cpu: + delta: true + enabled: true relabel_configs: - action: keep regex: true @@ -536,10 +535,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -591,6 +586,9 @@ data: enabled: false mutex: enabled: false + process_cpu: + delta: true + enabled: true relabel_configs: - action: keep regex: true @@ -666,10 +664,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -798,10 +792,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -930,10 +920,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -1053,7 +1039,7 @@ kind: ClusterRole metadata: name: default-phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1079,7 +1065,7 @@ kind: ClusterRoleBinding metadata: name: default-phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1167,7 +1153,7 @@ kind: Service metadata: name: phlare-dev-memberlist labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1193,7 +1179,7 @@ kind: Service metadata: name: phlare-dev-agent labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1217,7 +1203,7 @@ kind: Service metadata: name: phlare-dev-agent-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1242,7 +1228,7 @@ kind: Service metadata: name: phlare-dev-distributor labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1266,7 +1252,7 @@ kind: Service metadata: name: phlare-dev-distributor-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1291,7 +1277,7 @@ kind: Service metadata: name: phlare-dev-ingester labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1315,7 +1301,7 @@ kind: Service metadata: name: phlare-dev-ingester-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1340,7 +1326,7 @@ kind: Service metadata: name: phlare-dev-querier labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1364,7 +1350,7 @@ kind: Service metadata: name: phlare-dev-querier-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1389,7 +1375,7 @@ kind: Service metadata: name: phlare-dev-query-frontend labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1413,7 +1399,7 @@ kind: Service metadata: name: phlare-dev-query-frontend-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1438,7 +1424,7 @@ kind: Service metadata: name: phlare-dev-query-scheduler labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1462,7 +1448,7 @@ kind: Service metadata: name: phlare-dev-query-scheduler-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1487,7 +1473,7 @@ kind: Deployment metadata: name: phlare-dev-agent labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1503,9 +1489,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare @@ -1573,7 +1562,7 @@ kind: Deployment metadata: name: phlare-dev-distributor labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1589,9 +1578,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare @@ -1658,7 +1650,7 @@ kind: Deployment metadata: name: phlare-dev-querier labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1674,9 +1666,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare @@ -1743,7 +1738,7 @@ kind: Deployment metadata: name: phlare-dev-query-frontend labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1759,9 +1754,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare @@ -1828,7 +1826,7 @@ kind: Deployment metadata: name: phlare-dev-query-scheduler labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -1844,9 +1842,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare @@ -2008,7 +2009,7 @@ kind: StatefulSet metadata: name: phlare-dev-ingester labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -2026,9 +2027,12 @@ spec: template: metadata: annotations: - checksum/config: 65a7e6c11941c8c31ed1f5d154a82ad492d607df2707cb13b1fd1cbea3b1c1f5 + checksum/config: 1115b23a6b64ac92648f9e4bc43a571f388662fa077280488b506475ab51d69d + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare diff --git a/operations/phlare/helm/phlare/rendered/single-binary.yaml b/operations/phlare/helm/phlare/rendered/single-binary.yaml index e0fb746d7..2fb024a01 100644 --- a/operations/phlare/helm/phlare/rendered/single-binary.yaml +++ b/operations/phlare/helm/phlare/rendered/single-binary.yaml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: name: phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -17,7 +17,7 @@ kind: ConfigMap metadata: name: phlare-dev-overrides-config labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -33,7 +33,7 @@ kind: ConfigMap metadata: name: phlare-dev-config labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -65,10 +65,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -186,6 +182,9 @@ data: enabled: false mutex: enabled: false + process_cpu: + delta: true + enabled: true relabel_configs: - action: keep regex: true @@ -195,10 +194,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -250,6 +245,9 @@ data: enabled: false mutex: enabled: false + process_cpu: + delta: true + enabled: true relabel_configs: - action: keep regex: true @@ -325,10 +323,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -457,10 +451,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -589,10 +579,6 @@ data: regex: "" source_labels: - __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name - - action: keep - regex: (.*-(metrics|profiles)|http2) - source_labels: - - __meta_kubernetes_pod_container_port_name - action: replace regex: (https?) replacement: $1 @@ -704,7 +690,7 @@ kind: ClusterRole metadata: name: default-phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -730,7 +716,7 @@ kind: ClusterRoleBinding metadata: name: default-phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -750,7 +736,7 @@ kind: Service metadata: name: phlare-dev-memberlist labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -776,7 +762,7 @@ kind: Service metadata: name: phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -800,7 +786,7 @@ kind: Service metadata: name: phlare-dev-headless labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -825,7 +811,7 @@ kind: StatefulSet metadata: name: phlare-dev labels: - helm.sh/chart: phlare-0.5.2 + helm.sh/chart: phlare-0.5.3 app.kubernetes.io/name: phlare app.kubernetes.io/instance: phlare-dev app.kubernetes.io/version: "0.5.1" @@ -843,9 +829,12 @@ spec: template: metadata: annotations: - checksum/config: 900951ae5419296c60dba7db00d74498cf782f19fc2e006bf8c2f12b24f2bab7 + checksum/config: a781867cc5e9bddb8a630ac1d270dfc07fd37bfa5f566f550419e23902522266 + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/memory.scrape: "true" labels: app.kubernetes.io/name: phlare diff --git a/operations/phlare/helm/phlare/values.yaml b/operations/phlare/helm/phlare/values.yaml index 70b3b170b..72bd2488c 100644 --- a/operations/phlare/helm/phlare/values.yaml +++ b/operations/phlare/helm/phlare/values.yaml @@ -40,8 +40,11 @@ phlare: podAnnotations: # Scrapes itself see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles profiles.grafana.com/memory.scrape: "true" + profiles.grafana.com/memory.port_name: http2 profiles.grafana.com/cpu.scrape: "true" + profiles.grafana.com/cpu.port_name: http2 profiles.grafana.com/goroutine.scrape: "true" + profiles.grafana.com/goroutine.port_name: http2 # profiles.grafana.com/block.scrape: "true" # profiles.grafana.com/mutex.scrape: "true" @@ -186,9 +189,6 @@ phlare: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name] action: keep regex: '' - - source_labels: ['__meta_kubernetes_pod_container_port_name'] - action: 'keep' - regex: '(.*-(metrics|profiles)|http2)' - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme'] action: 'replace' target_label: '__scheme__' @@ -287,6 +287,9 @@ phlare: enabled: false goroutine: enabled: false + process_cpu: + enabled: true + delta: true relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape] action: keep @@ -294,9 +297,6 @@ phlare: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name] action: keep regex: '' - - source_labels: ['__meta_kubernetes_pod_container_port_name'] - action: 'keep' - regex: '(.*-(metrics|profiles)|http2)' - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme'] action: 'replace' target_label: '__scheme__' @@ -340,6 +340,9 @@ phlare: enabled: false goroutine: enabled: false + process_cpu: + enabled: true + delta: true relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape] action: keep @@ -402,9 +405,6 @@ phlare: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name] action: keep regex: '' - - source_labels: ['__meta_kubernetes_pod_container_port_name'] - action: 'keep' - regex: '(.*-(metrics|profiles)|http2)' - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme'] action: 'replace' target_label: '__scheme__' @@ -512,9 +512,6 @@ phlare: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name] action: keep regex: '' - - source_labels: ['__meta_kubernetes_pod_container_port_name'] - action: 'keep' - regex: '(.*-(metrics|profiles)|http2)' - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme'] action: 'replace' target_label: '__scheme__' @@ -622,9 +619,6 @@ phlare: - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name] action: keep regex: '' - - source_labels: ['__meta_kubernetes_pod_container_port_name'] - action: 'keep' - regex: '(.*-(metrics|profiles)|http2)' - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme'] action: 'replace' target_label: '__scheme__' diff --git a/operations/phlare/jsonnet/values.json b/operations/phlare/jsonnet/values.json index 6d5e2f7b3..61c93939a 100644 --- a/operations/phlare/jsonnet/values.json +++ b/operations/phlare/jsonnet/values.json @@ -32,7 +32,7 @@ "phlare": { "affinity": {}, "components": {}, - "config": "{{- if .Values.minio.enabled }}\nstorage:\n backend: s3\n s3:\n endpoint: \"{{ include \"phlare.fullname\" . }}-minio:9000\"\n bucket_name: {{(index .Values.minio.buckets 0).name | quote }}\n access_key_id: {{ .Values.minio.rootUser | quote }}\n secret_access_key: {{ .Values.minio.rootPassword | quote }}\n insecure: true\n{{- end }}\n# The default scrape config allows to define annotations based scraping.\n#\n# For example the following annotations:\n#\n# ```\n# profiles.grafana.com/memory.scrape: \"true\"\n# profiles.grafana.com/memory.port: \"8080\"\n# profiles.grafana.com/cpu.scrape: \"true\"\n# profiles.grafana.com/cpu.port: \"8080\"\n# profiles.grafana.com/goroutine.scrape: \"true\"\n# profiles.grafana.com/goroutine.port: \"8080\"\n# ```\n#\n# will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port of the pod.\n#\n# For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\nscrape_configs:\n - job_name: 'kubernetes-pods-memory-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: true\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n action: 'keep'\n regex: '(.*-(metrics|profiles)|http2)'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-memory-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: true\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-cpu-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n action: 'keep'\n regex: '(.*-(metrics|profiles)|http2)'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-cpu-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-block-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: true\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n action: 'keep'\n regex: '(.*-(metrics|profiles)|http2)'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-block-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: true\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-mutex-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: true\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n action: 'keep'\n regex: '(.*-(metrics|profiles)|http2)'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-mutex-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: true\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-goroutine-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: true\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n action: 'keep'\n regex: '(.*-(metrics|profiles)|http2)'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-goroutine-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: true\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n", + "config": "{{- if .Values.minio.enabled }}\nstorage:\n backend: s3\n s3:\n endpoint: \"{{ include \"phlare.fullname\" . }}-minio:9000\"\n bucket_name: {{(index .Values.minio.buckets 0).name | quote }}\n access_key_id: {{ .Values.minio.rootUser | quote }}\n secret_access_key: {{ .Values.minio.rootPassword | quote }}\n insecure: true\n{{- end }}\n# The default scrape config allows to define annotations based scraping.\n#\n# For example the following annotations:\n#\n# ```\n# profiles.grafana.com/memory.scrape: \"true\"\n# profiles.grafana.com/memory.port: \"8080\"\n# profiles.grafana.com/cpu.scrape: \"true\"\n# profiles.grafana.com/cpu.port: \"8080\"\n# profiles.grafana.com/goroutine.scrape: \"true\"\n# profiles.grafana.com/goroutine.port: \"8080\"\n# ```\n#\n# will scrape the `memory`, `cpu` and `goroutine` profiles from the `8080` port of the pod.\n#\n# For more information see https://grafana.com/docs/phlare/latest/operators-guide/deploy-kubernetes/#optional-scrape-your-own-workloads-profiles\nscrape_configs:\n - job_name: 'kubernetes-pods-memory-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: true\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-memory-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: true\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_memory_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_memory_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-cpu-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: true\n delta: true\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-cpu-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: true\n delta: true\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_cpu_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-block-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: true\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-block-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: true\n mutex:\n enabled: false\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_block_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_block_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_block_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-mutex-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: true\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-mutex-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: true\n goroutine:\n enabled: false\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_mutex_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-goroutine-default-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: true\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name]\n action: keep\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n - job_name: 'kubernetes-pods-goroutine-custom-name'\n scrape_interval: \"15s\"\n kubernetes_sd_configs:\n - role: pod\n profiling_config:\n pprof_config:\n memory:\n enabled: false\n block:\n enabled: false\n mutex:\n enabled: false\n goroutine:\n enabled: true\n process_cpu:\n enabled: false\n relabel_configs:\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scrape]\n action: keep\n regex: true\n - source_labels: [__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name]\n action: drop\n regex: ''\n - source_labels: ['__meta_kubernetes_pod_container_port_name']\n target_label: '__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port_name'\n action: 'keepequal'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_scheme']\n action: 'replace'\n target_label: '__scheme__'\n regex: '(https?)'\n replacement: '$1'\n - source_labels: ['__meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_path']\n action: 'replace'\n target_label: '__profile_path__'\n regex: '(.+)'\n replacement: '$1'\n - source_labels: [__address__, __meta_kubernetes_pod_annotation_profiles_grafana_com_goroutine_port]\n action: replace\n regex: (.+?)(?::\\d+)?;(\\d+)\n replacement: $1:$2\n target_label: __address__\n - action: labelmap\n regex: __meta_kubernetes_pod_label_(.+)\n - source_labels: [__meta_kubernetes_namespace]\n action: replace\n target_label: namespace\n - source_labels: [__meta_kubernetes_pod_name]\n action: replace\n target_label: pod\n - source_labels: ['__meta_kubernetes_pod_container_name']\n action: 'replace'\n target_label: 'container'\n - source_labels: [__meta_kubernetes_pod_phase]\n regex: Pending|Succeeded|Failed|Completed\n action: drop\n", "extraArgs": { "log.level": "debug" }, @@ -63,8 +63,11 @@ "size": "10Gi" }, "podAnnotations": { + "profiles.grafana.com/cpu.port_name": "http2", "profiles.grafana.com/cpu.scrape": "true", + "profiles.grafana.com/goroutine.port_name": "http2", "profiles.grafana.com/goroutine.scrape": "true", + "profiles.grafana.com/memory.port_name": "http2", "profiles.grafana.com/memory.scrape": "true" }, "podSecurityContext": { diff --git a/operations/phlare/relabeling_test.go b/operations/phlare/relabeling_test.go new file mode 100644 index 000000000..735628371 --- /dev/null +++ b/operations/phlare/relabeling_test.go @@ -0,0 +1,200 @@ +package phlare + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/grafana/phlare/pkg/agent" +) + +func Test_Relabeling(t *testing.T) { + // This tests the expected behavior of the relabeling rules in the helm chart. + fdata, err := os.Open("helm/phlare/rendered/single-binary.yaml") + require.NoError(t, err) + defer fdata.Close() + + values := map[string]interface{}{} + configString := `` + dec := yaml.NewDecoder(fdata) + for dec.Decode(&values) == nil { + if values["metadata"].(map[string]interface{})["name"] == "phlare-dev-config" { + configString = values["data"].(map[string]interface{})["config.yaml"].(string) + break + } + } + type Config struct { + ScrapeConfigs []agent.ScrapeConfig `yaml:"scrape_configs"` + } + cfg := &Config{} + require.NoError(t, yaml.Unmarshal([]byte(configString), cfg)) + + groups := []*agent.TargetGroup{} + for _, scrapeCfg := range cfg.ScrapeConfigs { + scrapeCfg.ScrapeInterval = model.Duration(15 * time.Second) + scrapeCfg.ScrapeTimeout = model.Duration(30 * time.Second) + groups = append(groups, agent.NewTargetGroup(context.Background(), "test", scrapeCfg, nil, "test", log.NewLogfmtLogger(os.Stdout))) + } + + for _, ty := range []string{"memory", "cpu", "block", "mutex", "goroutine"} { + ty := ty + for _, tt := range []struct { + name string + in []model.LabelSet + expectedActived []labels.Labels + }{ + { + "no targets", + []model.LabelSet{}, + nil, + }, + { + "default port", + []model.LabelSet{ + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + "__address__": "foo:1234", + }, + }, + []labels.Labels{ + { + labels.Label{Name: "instance", Value: "foo:1234"}, + labels.Label{Name: "job", Value: fmt.Sprintf("kubernetes-pods-%s-default-name", ty)}, + }, + }, + }, + { + "overrides port", + []model.LabelSet{ + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port", ty): "4100", + + "__address__": "foo:1234", + }, + }, + []labels.Labels{ + { + labels.Label{Name: "instance", Value: "foo:4100"}, + labels.Label{Name: "job", Value: fmt.Sprintf("kubernetes-pods-%s-default-name", ty)}, + }, + }, + }, + { + "overrides missing port", + []model.LabelSet{ + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port", ty): "4100", + + "__address__": "foo", + }, + }, + []labels.Labels{ + { + labels.Label{Name: "instance", Value: "foo:4100"}, + labels.Label{Name: "job", Value: fmt.Sprintf("kubernetes-pods-%s-default-name", ty)}, + }, + }, + }, + { + "custom port name ", + []model.LabelSet{ + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port_name", ty): "http2", + + "__meta_kubernetes_pod_container_port_name": "http2", + "__address__": "foo:123", + }, + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port", ty): "4200", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port_name", ty): "http2", + + "__meta_kubernetes_pod_container_port_name": "http", + "__address__": "foo:123", + }, + }, + []labels.Labels{ + { + labels.Label{Name: "instance", Value: "foo:123"}, + labels.Label{Name: "job", Value: fmt.Sprintf("kubernetes-pods-%s-custom-name", ty)}, + }, + }, + }, + { + "custom port name overrides port", + []model.LabelSet{ + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port", ty): "4100", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port_name", ty): "http2", + + "__meta_kubernetes_pod_container_port_name": "http2", + "__address__": "foo:123", + }, + { + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_scrape", ty): "true", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port", ty): "4200", + labeNameF("__meta_kubernetes_pod_annotation_profiles_grafana_com_%s_port_name", ty): "http2", + + "__meta_kubernetes_pod_container_port_name": "http", + "__address__": "foo:123", + }, + }, + []labels.Labels{ + { + labels.Label{Name: "instance", Value: "foo:4100"}, + labels.Label{Name: "job", Value: fmt.Sprintf("kubernetes-pods-%s-custom-name", ty)}, + }, + }, + }, + } { + tt := tt + t.Run(fmt.Sprintf(tt.name+"_"+ty), func(t *testing.T) { + t.Parallel() + var ( + actives []labels.Labels + dropped []labels.Labels + ) + for _, tg := range groups { + a, d, err := tg.TargetsFromGroup(&targetgroup.Group{ + Targets: tt.in, + }) + require.NoError(t, err) + for _, ta := range a { + if len(ta.Target.Labels()) > 0 { + ty := ty + if ty == "cpu" { + ty = "process_cpu" + } + require.Equal(t, ty, ta.Target.DiscoveredLabels().Get(model.MetricNameLabel)) + actives = append(actives, ta.Target.Labels()) + } + } + for _, t := range d { + if len(t.Target.Labels()) > 0 { + dropped = append(dropped, t.Target.Labels()) + } + } + } + + require.Equal(t, tt.expectedActived, actives, " mismatched actives dropped %v", dropped) + }) + } + } +} + +func labeNameF(format string, ty string) model.LabelName { + return model.LabelName(fmt.Sprintf(format, ty)) +} diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9c17c56fe..f453d6910 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -93,6 +93,7 @@ func (a *Agent) ActiveTargets() map[string][]*Target { for _, target := range tg.activeTargets { result[g] = append(result[g], target) } + tg.mtx.RUnlock() } return result } @@ -103,6 +104,7 @@ func (a *Agent) DroppedTargets() []*Target { for _, tg := range a.groups { tg.mtx.RLock() result = append(result, tg.droppedTargets...) + tg.mtx.RUnlock() } return result } diff --git a/pkg/agent/config.go b/pkg/agent/config.go index 4e7ded091..e05cec42e 100644 --- a/pkg/agent/config.go +++ b/pkg/agent/config.go @@ -128,7 +128,7 @@ func (c *ScrapeConfig) Validate() error { c.ScrapeTimeout = c.ScrapeInterval + model.Duration(3*time.Second) } if c.ScrapeTimeout <= c.ScrapeInterval { - return fmt.Errorf("scrape timeout must be larger or equal to interval for: %v", c.JobName) + return fmt.Errorf("scrape timeout must be larger scrape to interval for: %v", c.JobName) } if cfg, ok := c.ProfilingConfig.PprofConfig[pprofProcessCPU]; ok { diff --git a/pkg/agent/profiles.go b/pkg/agent/profiles.go index 95f2817a3..51c82fcda 100644 --- a/pkg/agent/profiles.go +++ b/pkg/agent/profiles.go @@ -99,10 +99,6 @@ func populateLabels(lset labels.Labels, cfg ScrapeConfig) (res, orig labels.Labe return nil, nil, errors.New("no address") } - if v := lset.Get(model.AddressLabel); v == "" { - return nil, nil, fmt.Errorf("no address") - } - lb = labels.NewBuilder(lset) // addPort checks whether we should add a default port to the address. @@ -154,8 +150,8 @@ func populateLabels(lset labels.Labels, cfg ScrapeConfig) (res, orig labels.Labe return nil, nil, errors.New("scrape timeout cannot be 0") } - if timeoutDuration > intervalDuration { - return nil, nil, fmt.Errorf("scrape timeout cannot be greater than scrape interval (%q > %q)", timeout, interval) + if timeoutDuration <= intervalDuration { + return nil, nil, fmt.Errorf("scrape timeout must be larger than scrape interval for (%q <= %q)", timeout, interval) } // Meta labels are deleted after relabelling. Other internal labels propagate to @@ -188,11 +184,17 @@ func intervalAndTimeoutFromLabels( defaultInterval, defaultDuration time.Duration, ) (time.Duration, time.Duration, error) { intervalLabel := labels.Get(model.ScrapeIntervalLabel) + if intervalLabel == "" { + intervalLabel = defaultInterval.String() + } interval, err := model.ParseDuration(intervalLabel) if err != nil { return defaultInterval, defaultDuration, fmt.Errorf("error parsing interval label %q: %v", intervalLabel, err) } timeoutLabel := labels.Get(model.ScrapeTimeoutLabel) + if timeoutLabel == "" { + timeoutLabel = defaultDuration.String() + } timeout, err := model.ParseDuration(timeoutLabel) if err != nil { return defaultInterval, defaultDuration, fmt.Errorf("error parsing timeout label %q: %v", timeoutLabel, err) @@ -201,8 +203,8 @@ func intervalAndTimeoutFromLabels( return time.Duration(interval), time.Duration(timeout), nil } -// targetsFromGroup builds targets based on the given TargetGroup and config. -func (tg *TargetGroup) targetsFromGroup(group *targetgroup.Group) ([]*Target, []*Target, error) { +// TargetsFromGroup builds targets based on the given TargetGroup and config. +func (tg *TargetGroup) TargetsFromGroup(group *targetgroup.Group) ([]*Target, []*Target, error) { var ( targets = make([]*Target, 0, len(group.Targets)) droppedTargets = make([]*Target, 0, len(group.Targets)) @@ -287,7 +289,7 @@ func (tg *TargetGroup) targetsFromGroup(group *targetgroup.Group) ([]*Target, [] } if pcfg, found := tg.config.ProfilingConfig.PprofConfig[profType]; found && pcfg.Delta { - params.Add("seconds", strconv.Itoa(int(time.Duration(tg.config.ScrapeInterval)/time.Second)-1)) + params.Add("seconds", strconv.Itoa(int(interval/time.Second)-1)) } targets = append(targets, &Target{ Target: scrape.NewTarget(lbls, origLabels, params), diff --git a/pkg/agent/profiles_test.go b/pkg/agent/profiles_test.go index 263891a58..89f5c48ba 100644 --- a/pkg/agent/profiles_test.go +++ b/pkg/agent/profiles_test.go @@ -31,7 +31,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: labels.FromMap(map[string]string{ model.AddressLabel: "1.2.3.4:1000", @@ -39,7 +39,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", "custom": "value", }), wantOrig: labels.FromMap(map[string]string{ @@ -47,7 +47,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", "custom": "value", }), wantErr: false, @@ -59,13 +59,13 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "http", model.JobLabel: "custom-job", model.ScrapeIntervalLabel: "2s", - model.ScrapeTimeoutLabel: "2s", + model.ScrapeTimeoutLabel: "3s", }), cfg: ScrapeConfig{ Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: labels.FromMap(map[string]string{ model.AddressLabel: "1.2.3.4:80", @@ -73,14 +73,14 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "http", model.JobLabel: "custom-job", model.ScrapeIntervalLabel: "2s", - model.ScrapeTimeoutLabel: "2s", + model.ScrapeTimeoutLabel: "3s", }), wantOrig: labels.FromMap(map[string]string{ model.AddressLabel: "1.2.3.4", model.SchemeLabel: "http", model.JobLabel: "custom-job", model.ScrapeIntervalLabel: "2s", - model.ScrapeTimeoutLabel: "2s", + model.ScrapeTimeoutLabel: "3s", }), wantErr: false, }, @@ -94,7 +94,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: labels.FromMap(map[string]string{ model.AddressLabel: "[::1]:443", @@ -102,7 +102,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", }), wantOrig: labels.FromMap(map[string]string{ model.AddressLabel: "[::1]", @@ -110,7 +110,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", }), wantErr: false, }, @@ -121,7 +121,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: nil, wantOrig: nil, @@ -134,7 +134,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), RelabelConfigs: []*relabel.Config{ { Action: relabel.Replace, @@ -151,7 +151,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", "custom": "host:1234", }), wantOrig: labels.FromMap(map[string]string{ @@ -159,7 +159,7 @@ func TestPopulateLabels(t *testing.T) { model.SchemeLabel: "https", model.JobLabel: "job", model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "1s", + model.ScrapeTimeoutLabel: "2s", "custom": "host:1234", }), }, @@ -173,7 +173,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: nil, wantOrig: nil, @@ -189,7 +189,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: nil, wantOrig: nil, @@ -205,7 +205,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: nil, wantOrig: nil, @@ -237,7 +237,7 @@ func TestPopulateLabels(t *testing.T) { Scheme: "https", JobName: "job", ScrapeInterval: model.Duration(time.Second), - ScrapeTimeout: model.Duration(time.Second), + ScrapeTimeout: 2 * model.Duration(time.Second), }, wantRes: nil, wantOrig: nil, @@ -247,13 +247,13 @@ func TestPopulateLabels(t *testing.T) { name: "timeout less than interval", labels: labels.FromMap(map[string]string{ model.AddressLabel: "1.2.3.4:1000", - model.ScrapeIntervalLabel: "1s", - model.ScrapeTimeoutLabel: "2s", + model.ScrapeIntervalLabel: "2s", + model.ScrapeTimeoutLabel: "1s", }), cfg: ScrapeConfig{ Scheme: "https", JobName: "job", - ScrapeInterval: model.Duration(time.Second), + ScrapeInterval: 2 * model.Duration(time.Second), ScrapeTimeout: model.Duration(time.Second), }, wantRes: nil, @@ -289,7 +289,7 @@ func TestTargetGroup_targetsFromGroup(t *testing.T) { tg: &TargetGroup{ jobName: "job", config: ScrapeConfig{ - ScrapeTimeout: model.Duration(10 * time.Second), + ScrapeTimeout: model.Duration(10 * time.Minute), ScrapeInterval: model.Duration(time.Minute), ProfilingConfig: &config.ProfilingConfig{ PprofConfig: config.PprofConfig{ @@ -331,7 +331,7 @@ func TestTargetGroup_targetsFromGroup(t *testing.T) { }, Labels: model.LabelSet{ model.ScrapeIntervalLabel: "30s", - model.ScrapeTimeoutLabel: "20s", + model.ScrapeTimeoutLabel: "40s", }, }, wantTargets: 1, @@ -341,7 +341,7 @@ func TestTargetGroup_targetsFromGroup(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotTargets, gotDropped, err := tt.tg.targetsFromGroup(tt.group) + gotTargets, gotDropped, err := tt.tg.TargetsFromGroup(tt.group) if tt.wantErr { require.NoError(t, err) } else { diff --git a/pkg/agent/target.go b/pkg/agent/target.go index a6ccac9a3..4c54d2959 100644 --- a/pkg/agent/target.go +++ b/pkg/agent/target.go @@ -74,7 +74,7 @@ func (tg *TargetGroup) sync(groups []*targetgroup.Group) { var actives []*Target tg.droppedTargets = []*Target{} for _, group := range groups { - targets, dropped, err := tg.targetsFromGroup(group) + targets, dropped, err := tg.TargetsFromGroup(group) if err != nil { level.Error(tg.logger).Log("msg", "creating targets failed", "err", err) continue