feat(obs): Improve builtin grafana setup to have traces and logs link…

…ed (#2716) - Add datasources for jaeger and loki in `metrics install`. It simplifies life after and doesn't cause any problems if there's no jaeger installed. - Add traces to access log format - Extract correct timestamp, mesh and add node_id as a key in promtail - Add links between traces and access logs in grafana. Signed-off-by: Charly Molter <charly.molter@konghq.com> (cherry picked from commit dfe1e38)
kumahq · Sep 2, 2021 · 7eea202 · 7eea202
1 parent 6395917
commit 7eea202
Show file tree

Hide file tree

Showing 15 changed files with 228 additions and 4 deletions.
diff --git a/app/kumactl/cmd/completion/testdata/bash.golden b/app/kumactl/cmd/completion/testdata/bash.golden
@@ -2956,10 +2956,18 @@ _kumactl_install_metrics()
     flags_with_completion=()
     flags_completion=()
 
+    flags+=("--jaeger-address=")
+    two_word_flags+=("--jaeger-address")
+    local_nonpersistent_flags+=("--jaeger-address")
+    local_nonpersistent_flags+=("--jaeger-address=")
     flags+=("--kuma-cp-address=")
     two_word_flags+=("--kuma-cp-address")
     local_nonpersistent_flags+=("--kuma-cp-address")
     local_nonpersistent_flags+=("--kuma-cp-address=")
+    flags+=("--loki-address=")
+    two_word_flags+=("--loki-address")
+    local_nonpersistent_flags+=("--loki-address")
+    local_nonpersistent_flags+=("--loki-address=")
     flags+=("--namespace=")
     two_word_flags+=("--namespace")
     local_nonpersistent_flags+=("--namespace")

diff --git a/app/kumactl/cmd/install/context/install_metrics_context.go b/app/kumactl/cmd/install/context/install_metrics_context.go
@@ -9,6 +9,8 @@ type MetricsTemplateArgs struct {
 	Namespace         string
 	Mesh              string
 	KumaCpAddress     string
+	JaegerAddress     string
+	LokiAddress       string
 	KumaCpApiAddress  string
 	WithoutPrometheus bool
 	WithoutGrafana    bool
@@ -25,6 +27,8 @@ func DefaultInstallMetricsContext() InstallMetricsContext {
 			Namespace:         "kuma-metrics",
 			KumaCpAddress:     "http://kuma-control-plane.kuma-system:5676",
 			KumaCpApiAddress:  "http://kuma-control-plane.kuma-system:5681",
+			JaegerAddress:     "http://jaeger-query.kuma-tracing",
+			LokiAddress:       "http://loki.kuma-logging:3100",
 			WithoutPrometheus: false,
 			WithoutGrafana:    false,
 		},

diff --git a/app/kumactl/cmd/install/install_metrics.go b/app/kumactl/cmd/install/install_metrics.go
@@ -98,6 +98,8 @@ func newInstallMetrics(pctx *kumactl_cmd.RootContext) *cobra.Command {
 	}
 	cmd.Flags().StringVar(&args.Namespace, "namespace", args.Namespace, "namespace to install metrics to")
 	cmd.Flags().StringVar(&args.KumaCpAddress, "kuma-cp-address", args.KumaCpAddress, "the address of Kuma CP")
+	cmd.Flags().StringVar(&args.JaegerAddress, "jaeger-address", args.JaegerAddress, "the address of jaeger to query")
+	cmd.Flags().StringVar(&args.LokiAddress, "loki-address", args.LokiAddress, "the address of the loki to query")
 	cmd.Flags().BoolVar(&args.WithoutPrometheus, "without-prometheus", args.WithoutPrometheus, "disable Prometheus resources generation")
 	cmd.Flags().BoolVar(&args.WithoutGrafana, "without-grafana", args.WithoutGrafana, "disable Grafana resources generation")
 	return cmd

diff --git a/app/kumactl/cmd/install/testdata/install-logging.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-logging.defaults.golden.yaml
@@ -137,9 +137,27 @@ data:
     - job_name: kubernetes-pods-name
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - source_labels:
         - __meta_kubernetes_pod_label_name
         target_label: __service__
@@ -180,9 +198,27 @@ data:
     - job_name: kubernetes-pods-app
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - action: drop
         regex: .+
         source_labels:

diff --git a/app/kumactl/cmd/install/testdata/install-logging.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-logging.overrides.golden.yaml
@@ -137,9 +137,27 @@ data:
     - job_name: kubernetes-pods-name
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - source_labels:
         - __meta_kubernetes_pod_label_name
         target_label: __service__
@@ -180,9 +198,27 @@ data:
     - job_name: kubernetes-pods-app
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - action: drop
         regex: .+
         source_labels:

diff --git a/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml
@@ -175,6 +175,7 @@ data:
     datasources:
       # <string, required> name of the datasource. Required
       - name: Prometheus
+        uid: prometheus
         # <string, required> datasource type. Required
         type: prometheus
         # <string, required> access mode. direct or proxy. Required
@@ -213,10 +214,34 @@ data:
         # <bool> allow users to edit datasources from the UI.
         editable: true
       - name: Kuma
+        uid: kuma
+        editable: true
         type: kumahq-kuma-datasource
         url: http://kuma-control-plane.kuma-system:5681
         jsonData:
           prometheusDataSourceId: "1"
+      - name: Jaeger
+        type: jaeger
+        access: proxy
+        editable: true
+        uid: jaeger
+        url: http://jaeger-query.kuma-tracing
+        jsonData:
+          tracesToLogs:
+            datasourceUid: loki
+            tags: ["node_id"]
+      - name: Loki
+        type: loki
+        access: proxy
+        editable: true
+        uid: loki
+        url: http://loki.kuma-logging:3100
+        jsonData:
+          derivedFields:
+            - datasourceUid: jaeger
+              matcherRegex: '"([0-9a-f]{16})"'
+              name: "traceId"
+              url: '$${__value.raw}'
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml
@@ -111,6 +111,7 @@ data:
     datasources:
       # <string, required> name of the datasource. Required
       - name: Prometheus
+        uid: prometheus
         # <string, required> datasource type. Required
         type: prometheus
         # <string, required> access mode. direct or proxy. Required
@@ -149,10 +150,34 @@ data:
         # <bool> allow users to edit datasources from the UI.
         editable: true
       - name: Kuma
+        uid: kuma
+        editable: true
         type: kumahq-kuma-datasource
         url: http://kuma-control-plane.kuma-system:5681
         jsonData:
           prometheusDataSourceId: "1"
+      - name: Jaeger
+        type: jaeger
+        access: proxy
+        editable: true
+        uid: jaeger
+        url: http://jaeger-query.kuma-tracing
+        jsonData:
+          tracesToLogs:
+            datasourceUid: loki
+            tags: ["node_id"]
+      - name: Loki
+        type: loki
+        access: proxy
+        editable: true
+        uid: loki
+        url: http://loki.kuma-logging:3100
+        jsonData:
+          derivedFields:
+            - datasourceUid: jaeger
+              matcherRegex: '"([0-9a-f]{16})"'
+              name: "traceId"
+              url: '$${__value.raw}'
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml
@@ -175,6 +175,7 @@ data:
     datasources:
       # <string, required> name of the datasource. Required
       - name: Prometheus
+        uid: prometheus
         # <string, required> datasource type. Required
         type: prometheus
         # <string, required> access mode. direct or proxy. Required
@@ -213,10 +214,34 @@ data:
         # <bool> allow users to edit datasources from the UI.
         editable: true
       - name: Kuma
+        uid: kuma
+        editable: true
         type: kumahq-kuma-datasource
         url: http://kuma-control-plane.kuma-system:5681
         jsonData:
           prometheusDataSourceId: "1"
+      - name: Jaeger
+        type: jaeger
+        access: proxy
+        editable: true
+        uid: jaeger
+        url: http://jaeger-query.kuma-tracing
+        jsonData:
+          tracesToLogs:
+            datasourceUid: loki
+            tags: ["node_id"]
+      - name: Loki
+        type: loki
+        access: proxy
+        editable: true
+        uid: loki
+        url: http://loki.kuma-logging:3100
+        jsonData:
+          derivedFields:
+            - datasourceUid: jaeger
+              matcherRegex: '"([0-9a-f]{16})"'
+              name: "traceId"
+              url: '$${__value.raw}'
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/app/kumactl/data/install/k8s/logging/loki/loki.yaml b/app/kumactl/data/install/k8s/logging/loki/loki.yaml
@@ -110,9 +110,27 @@ data:
     - job_name: kubernetes-pods-name
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - source_labels:
         - __meta_kubernetes_pod_label_name
         target_label: __service__
@@ -153,9 +171,27 @@ data:
     - job_name: kubernetes-pods-app
       pipeline_stages:
         - docker: {}
+        - match:
+            selector: '{container="kuma-sidecar"}'
+            stages:
+              - regex:
+                  expression: '([\S]*) stdout ([\S]*) \[(?P<timestamp>[\S]*)\] (?P<mesh>[\S]*)'
+              - labels:
+                  mesh: mesh
+              - timestamp:
+                  format: RFC3339Nano
+                  source: timestamp
       kubernetes_sd_configs:
       - role: pod
       relabel_configs:
+      - action: replace
+        separator: '.'
+        replacement: "$1"
+        source_labels:
+        - __meta_kubernetes_pod_annotation_kuma_io_mesh
+        - __meta_kubernetes_pod_name
+        - __meta_kubernetes_namespace
+        target_label: node_id
       - action: drop
         regex: .+
         source_labels:

diff --git a/app/kumactl/data/install/k8s/metrics/grafana/grafana.yaml b/app/kumactl/data/install/k8s/metrics/grafana/grafana.yaml
@@ -94,6 +94,7 @@ data:
     datasources:
       # <string, required> name of the datasource. Required
       - name: Prometheus
+        uid: prometheus
         # <string, required> datasource type. Required
         type: prometheus
         # <string, required> access mode. direct or proxy. Required
@@ -132,10 +133,34 @@ data:
         # <bool> allow users to edit datasources from the UI.
         editable: true
       - name: Kuma
+        uid: kuma
+        editable: true
         type: kumahq-kuma-datasource
         url: {{.KumaCpApiAddress}}
         jsonData:
           prometheusDataSourceId: "1"
+      - name: Jaeger
+        type: jaeger
+        access: proxy
+        editable: true
+        uid: jaeger
+        url: {{.JaegerAddress}}
+        jsonData:
+          tracesToLogs:
+            datasourceUid: loki
+            tags: ["node_id"]
+      - name: Loki
+        type: loki
+        access: proxy
+        editable: true
+        uid: loki
+        url: {{.LokiAddress}}
+        jsonData:
+          derivedFields:
+            - datasourceUid: jaeger
+              matcherRegex: '"([0-9a-f]{16})"'
+              name: "traceId"
+              url: '$${__value.raw}'
 ---
 apiVersion: v1
 kind: ConfigMap

diff --git a/docs/cmd/kumactl/kumactl_install_metrics.md b/docs/cmd/kumactl/kumactl_install_metrics.md
@@ -14,7 +14,9 @@ kumactl install metrics [flags]
 
 ```
   -h, --help                     help for metrics
+      --jaeger-address string    the address of jaeger to query (default "http://jaeger-query.kuma-tracing")
       --kuma-cp-address string   the address of Kuma CP (default "http://kuma-control-plane.kuma-system:5676")
+      --loki-address string      the address of the loki to query (default "http://loki.kuma-logging:3100")
       --namespace string         namespace to install metrics to (default "kuma-metrics")
       --without-grafana          disable Grafana resources generation
       --without-prometheus       disable Prometheus resources generation