Remove the default metrics namespace/prefix for span RED metrics in J…

…aeger query (#1072) * queryfrontend: remove default RED metric namespace Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de> * tempostack: expose option to set RED Metrics namespace Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de> * switch to breaking change senario Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de> * use old otel span metrics prefix Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de> * generate Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de> --------- Signed-off-by: Benedikt Bongartz <bongartz@klimlive.de>
grafana · Oct 30, 2024 · e92ec34 · e92ec34
1 parent 762c39a
commit e92ec34
Show file tree

Hide file tree

Showing 13 changed files with 102 additions and 3 deletions.
diff --git a/.chloggen/fix_red-metrics.yaml b/.chloggen/fix_red-metrics.yaml
@@ -0,0 +1,41 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action)
+component: tempostack
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Use new default metrics namespace/prefix for span RED metrics in Jaeger query.
+
+
+# One or more tracking issues related to the change
+issues: [1072]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Use the new RED metrics default namespace `traces.span.metrics` for retrieval from Prometheus.
+  Since OpenTelemetry Collector version 0.109.0 the default namespace is set to traces.span.metrics.
+  The namespace taken into account by jaeger-query can be configured via a TempoStack CR entry.
+  To achieve this the Operator will set the jaeger-query `--prometheus.query.namespace=` flag.
+  Since Jaeger version 1.62, jaeger-query uses `traces.span.metrics` as default too.
+
+  Example how to restore the default namespace used prior to version `0.109.0`, by configuring an empty value for `redMetricsNamespace` in the TempoStack CR:
+  ```
+  apiVersion: tempo.grafana.com/v1alpha1
+  kind: TempoStack
+  ...
+  spec:
+    template:
+      queryFrontend:
+        jaegerQuery:
+          enabled: true
+          monitorTab:
+            enabled: true
+            prometheusEndpoint: "http://myPromInstance:9090"
+            redMetricsNamespace: ""
+  ```
+  More details can be found here:
+  - https://github.com/jaegertracing/jaeger/pull/6007
+  - https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34485
diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go
@@ -689,6 +689,13 @@ type JaegerQueryMonitor struct {
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint"
 	PrometheusEndpoint string `json:"prometheusEndpoint"`
+	// REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+	// By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+	// +optional
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:=traces.span.metrics
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace"
+	REDMetricsNamespace string `json:"redMetricsNamespace"`
 }
 
 // IngressSpec defines Jaeger Query Ingress options.

diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1070,6 +1070,12 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
+          following the default namespace of the OpenTelemetry Collector since Version
+          0.109.0.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml
@@ -2470,6 +2470,12 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                default: traces.span.metrics
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml
@@ -1070,6 +1070,12 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
+          following the default namespace of the OpenTelemetry Collector since Version
+          0.109.0.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml
@@ -2470,6 +2470,12 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                default: traces.span.metrics
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml
@@ -2466,6 +2466,12 @@ spec:
                                   PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics.
                                   For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
                                 type: string
+                              redMetricsNamespace:
+                                default: traces.span.metrics
+                                description: |-
+                                  REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics.
+                                  By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
+                                type: string
                             type: object
                           resources:
                             description: Resources defines resources for this component,

diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml
@@ -999,6 +999,12 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
+          following the default namespace of the OpenTelemetry Collector since Version
+          0.109.0.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml
@@ -999,6 +999,12 @@ spec:
           on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
         displayName: Prometheus endpoint
         path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint
+      - description: REDMetricsNamespace defines the a prefix used retrieve span rate,
+          error, and duration (RED) metrics. By default it is set to `traces.span.metrics`
+          following the default namespace of the OpenTelemetry Collector since Version
+          0.109.0.
+        displayName: RED Metric Namespace
+        path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace
       - description: Resources defines resources for this component, this will override
           the calculated resources derived from total
         displayName: Resources

diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml
@@ -342,6 +342,7 @@ spec:                                    # TempoStackSpec defines the desired st
         monitorTab:                      # MonitorTab defines the monitor tab configuration.
           enabled: false                 # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature.
           prometheusEndpoint: ""         # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
+          redMetricsNamespace: "traces.span.metrics" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0.
         servicesQueryDuration: ""        # ServicesQueryDuration defines how long the services will be available in the services list
         tempoQuery:                      # TempoQuery defines options specific to the Tempoo Query component.
           resources:                     # Resources defines resources for this component, this will override the calculated resources derived from total

diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go
@@ -398,6 +398,11 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1.
 			// However, we do not intend to support them.
 			// --prometheus.query.normalize-calls
 			// --prometheus.query.normalize-duration
+			//
+			// NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics".
+			// We fallback to no namespace.
+			// See https://github.com/jaegertracing/jaeger/pull/6007.
+			fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace),
 		},
 	}
 	// If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth

diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go
@@ -621,15 +621,16 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 							JaegerQuery: v1alpha1.JaegerQuerySpec{
 								Enabled: true,
 								MonitorTab: v1alpha1.JaegerQueryMonitor{
-									Enabled:            true,
-									PrometheusEndpoint: "http://prometheus:9091",
+									Enabled:             true,
+									PrometheusEndpoint:  "http://prometheus:9091",
+									REDMetricsNamespace: "test",
 								},
 							},
 						},
 					},
 				},
 			},
-			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true"},
+			args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"},
 			env:  []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}},
 		},
 		{
@@ -657,6 +658,7 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) {
 				"--span-storage.type=grpc",
 				"--grpc-storage.server=localhost:7777",
 				"--query.bearer-token-propagation=true",
+				"--prometheus.query.namespace=",
 				"--prometheus.tls.enabled=true",
 				"--prometheus.token-file=/var/run/secrets/kubernetes.io/serviceaccount/token",
 				"--prometheus.token-override-from-context=false",

diff --git a/tests/e2e-openshift/red-metrics/03-install-tempo.yaml b/tests/e2e-openshift/red-metrics/03-install-tempo.yaml
@@ -28,5 +28,6 @@ spec:
         monitorTab:
           enabled: true
           prometheusEndpoint: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091
+          redMetricsNamespace: ""
         ingress:
           type: route