From 00198ac5d59d8b96975b0d166fdf4f1dee755f45 Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Mon, 28 Feb 2022 19:00:49 +0100 Subject: [PATCH] tempo-mixin: add metrics-generator to Tempo / Resources and Operational (#1313) * tempo-mixin: add metrics-generator to Tempo / Resources * tempo-mixin: add metrics-generator to Tempo / Operational --- operations/tempo-mixin/config.libsonnet | 1 + .../dashboards/tempo-operational.json | 16 +- .../dashboards/tempo-resources.libsonnet | 12 + .../tempo-mixin/yamls/tempo-operational.json | 16 +- .../tempo-mixin/yamls/tempo-resources.json | 315 +++++++++++++++++- 5 files changed, 346 insertions(+), 14 deletions(-) diff --git a/operations/tempo-mixin/config.libsonnet b/operations/tempo-mixin/config.libsonnet index 6ae32b6f04a..e69a7b7504d 100644 --- a/operations/tempo-mixin/config.libsonnet +++ b/operations/tempo-mixin/config.libsonnet @@ -10,6 +10,7 @@ query_frontend: 'query-frontend', querier: 'querier', ingester: 'ingester', + metrics_generator: 'metrics-generator', distributor: 'distributor', compactor: 'compactor', }, diff --git a/operations/tempo-mixin/dashboards/tempo-operational.json b/operations/tempo-mixin/dashboards/tempo-operational.json index a404d1fcf61..9b0e3b3406e 100644 --- a/operations/tempo-mixin/dashboards/tempo-operational.json +++ b/operations/tempo-mixin/dashboards/tempo-operational.json @@ -5438,7 +5438,7 @@ "value": "tempo-ops" }, "datasource": "$ds", - "definition": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)", + "definition": "label_values(tempo_build_info, namespace)", "description": null, "error": null, "hide": 0, @@ -5448,7 +5448,7 @@ "name": "namespace", "options": [], "query": { - "query": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(tempo_build_info, namespace)", "refId": "ops-cortex-namespace-Variable-Query" }, "refresh": 1, @@ -5495,6 +5495,16 @@ "text": "ingester", "value": "ingester" }, + { + "selected": false, + "text": "metrics-generator", + "value": "metrics-generator" + }, + { + "selected": false, + "text": "query-frontend", + "value": "query-frontend" + }, { "selected": false, "text": "querier", @@ -5506,7 +5516,7 @@ "value": "cortex-gw" } ], - "query": "compactor,distributor,ingester,querier,cortex-gw", + "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier,cortex-gw", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/operations/tempo-mixin/dashboards/tempo-resources.libsonnet b/operations/tempo-mixin/dashboards/tempo-resources.libsonnet index 5d1b6c03ba7..f5b815ee647 100644 --- a/operations/tempo-mixin/dashboards/tempo-resources.libsonnet +++ b/operations/tempo-mixin/dashboards/tempo-resources.libsonnet @@ -42,6 +42,18 @@ dashboard_utils { $.goHeapInUsePanel('Memory (go heap inuse)', $.jobMatcher($._config.jobs.ingester)), ) ) + .addRow( + g.row('Metrics-generator') + .addPanel( + $.containerCPUUsagePanel('CPU', $._config.jobs.metrics_generator), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', $._config.jobs.metrics_generator), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', $.jobMatcher($._config.jobs.metrics_generator)), + ) + ) .addRow( g.row('Query Frontend') .addPanel( diff --git a/operations/tempo-mixin/yamls/tempo-operational.json b/operations/tempo-mixin/yamls/tempo-operational.json index 0ebf2f3aecc..1fae68dbf13 100644 --- a/operations/tempo-mixin/yamls/tempo-operational.json +++ b/operations/tempo-mixin/yamls/tempo-operational.json @@ -6040,7 +6040,7 @@ "value": "tempo-ops" }, "datasource": "$ds", - "definition": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)", + "definition": "label_values(tempo_build_info, namespace)", "description": null, "error": null, "hide": 0, @@ -6052,7 +6052,7 @@ ], "query": { - "query": "label_values(kube_pod_container_info{image=~\".*tempo.*\", cluster=\"$cluster\"}, namespace)", + "query": "label_values(tempo_build_info, namespace)", "refId": "ops-cortex-namespace-Variable-Query" }, "refresh": 1, @@ -6099,6 +6099,16 @@ "text": "ingester", "value": "ingester" }, + { + "selected": false, + "text": "metrics-generator", + "value": "metrics-generator" + }, + { + "selected": false, + "text": "query-frontend", + "value": "query-frontend" + }, { "selected": false, "text": "querier", @@ -6110,7 +6120,7 @@ "value": "cortex-gw" } ], - "query": "compactor,distributor,ingester,querier,cortex-gw", + "query": "compactor,distributor,ingester,metrics-generator,query-frontend,querier,cortex-gw", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/operations/tempo-mixin/yamls/tempo-resources.json b/operations/tempo-mixin/yamls/tempo-resources.json index bda621fed27..a5875f6e04e 100644 --- a/operations/tempo-mixin/yamls/tempo-resources.json +++ b/operations/tempo-mixin/yamls/tempo-resources.json @@ -948,6 +948,305 @@ "linewidth": 1, "links": [ + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"metrics-generator\"}[$__interval]))", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"metrics-generator\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=\"metrics-generator\"} > 0)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(instance) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/metrics-generator\"})", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Metrics-generator", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + ], "nullPointMode": "null as zero", "percentage": false, @@ -1034,7 +1333,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 11, + "id": 14, "legend": { "avg": false, "current": false, @@ -1134,7 +1433,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 12, + "id": 15, "legend": { "avg": false, "current": false, @@ -1233,7 +1532,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 16, "legend": { "avg": false, "current": false, @@ -1333,7 +1632,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 14, + "id": 17, "legend": { "avg": false, "current": false, @@ -1433,7 +1732,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 18, "legend": { "avg": false, "current": false, @@ -1532,7 +1831,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 16, + "id": 19, "legend": { "avg": false, "current": false, @@ -1632,7 +1931,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 20, "legend": { "avg": false, "current": false, @@ -1732,7 +2031,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 21, "legend": { "avg": false, "current": false,