[Metricbeat] gcp: allow service metric prefix override (#26960)

CHANGELOG.next.asciidoc

@@ -512,6 +512,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix memory leak in SQL module when database is not available. {issue}25840[25840] {pull}26607[26607]
 - Fix aws metric tags with resourcegroupstaggingapi paginator.  {issue}26385[26385] {pull}26443[26443]
 - Fix quoting in GCP billing table name  {issue}26855[26855] {pull}26870[26870]
+- Allow metric prefix override per service in gcp module. {pull}26960[26960]
 
 *Packetbeat*
 

metricbeat/docs/modules/gcp.asciidoc

@@ -286,6 +286,20 @@ metricbeat.modules:
         - "instance/cpu/utilization"
         - "instance/uptime"
 
+- module: gcp
+  metricsets:
+    - metrics
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  metrics:
+    - aligner: ALIGN_NONE
+      service: kubernetes
+      service_metric_prefix: kubernetes.io/
+      metric_types:
+        - "container/cpu/core_usage_time"
+
 - module: gcp
   metricsets:
     - billing

x-pack/metricbeat/metricbeat.reference.yml

@@ -569,6 +569,20 @@ metricbeat.modules:
         - "instance/cpu/utilization"
         - "instance/uptime"
 
+- module: gcp
+  metricsets:
+    - metrics
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  metrics:
+    - aligner: ALIGN_NONE
+      service: kubernetes
+      service_metric_prefix: kubernetes.io/
+      metric_types:
+        - "container/cpu/core_usage_time"
+
 - module: gcp
   metricsets:
     - billing

x-pack/metricbeat/module/gcp/_meta/config.yml

@@ -41,6 +41,20 @@
         - "instance/cpu/utilization"
         - "instance/uptime"
 
+- module: gcp
+  metricsets:
+    - metrics
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  metrics:
+    - aligner: ALIGN_NONE
+      service: kubernetes
+      service_metric_prefix: kubernetes.io/
+      metric_types:
+        - "container/cpu/core_usage_time"
+
 - module: gcp
   metricsets:
     - billing

x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc

@@ -16,12 +16,6 @@ call.
 [float]
 == Metricset config and parameters
 
-* *metric_types*: Required, a list of metric type strings, or a list of metric
-type prefixes. For example, `instance/cpu` is the prefix for metric type
-`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the
-`ListTimeSeries` API can return any number of time series from a single metric
-type. Metric type is to used for identifying a specific time series.
-
 * *aligner*: A single string with which aggregation operation need to be applied
 onto time series data for ListTimeSeries API. If it's not given, default aligner
 is `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`,
@@ -30,6 +24,27 @@ Please see
 https://cloud.google.com/monitoring/api/ref_v3/rpc/google.monitoring.v3#aligner[Aggregation Aligner]
 for the full list of aligners.
 
+* *metric_types*: Required, a list of metric type strings, or a list of metric
+type prefixes. For example, `instance/cpu` is the prefix for metric type
+`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the
+`ListTimeSeries` API can return any number of time series from a single metric
+type. Metric type is to used for identifying a specific time series.
+
+* *service*: Required, the name of the service for related metrics. This should
+be a valid Google Cloud service name. Service names may be viewed from the
+corresponding page from https://cloud.google.com/monitoring/api/metrics[GCP Metrics list documentation].
+The `service` field is used to compute the GCP Metric prefix, unless
+`service_metric_prefix` is set.
+
+* *service_metric_prefix*: A string containing the full Metric prefix as
+specified in the GCP documentation.
+All metrics from GCP Monitoring API require a prefix. When
+`service_metric_prefix` is empty, the prefix default to a value computed using
+the `service` value: `<service>.googleapis.com/`. This default works for any
+services under "Google Cloud metrics", but does not work for other services
+(`kubernetes` aka GKE for example).
+This option allow to override the default and specify an arbitrary metric prefix.
+
 [float]
 === Example Configuration
 * `metrics` metricset is enabled to collect metrics from all zones under
@@ -124,3 +139,27 @@ every minute with no aggregation. The metric types in `compute` service with
       metric_types:
         - "instance/cpu"
 ----
+
+* `metrics` metricset is enabled to collect metrics from all zones under
+`europe-west1-c` region in `elastic-observability` project. One set of metrics
+will be collected: core usage time for containers in GCP GKE.
+Note that the is required to use `service_metric_prefix` to override the default
+metric prefix, as for GKE metrics the required prefix is `kubernetes.io/`
+
++
+[source,yaml]
+----
+- module: googlecloud
+  metricsets:
+    - metrics
+  zone: "europe-west1-c"
+  project_id: elastic-observability
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  metrics:
+    - service: kubernetes
+      service_metric_prefix: kubernetes.io/
+      metric_types:
+        - "container/cpu/core_usage_time"
+----

x-pack/metricbeat/module/gcp/metrics/metricset.go

@@ -7,6 +7,7 @@ package metrics
 import (
 	"context"
 	"fmt"
+	"path"
 	"time"
 
 	monitoring "cloud.google.com/go/monitoring/apiv3"
@@ -51,9 +52,25 @@ type MetricSet struct {
 
 //metricsConfig holds a configuration specific for metrics metricset.
 type metricsConfig struct {
-	ServiceName string   `config:"service"  validate:"required"`
-	MetricTypes []string `config:"metric_types" validate:"required"`
-	Aligner     string   `config:"aligner"`
+	ServiceName string `config:"service"  validate:"required"`
+	// MetricPrefix allows to specify the prefix string for MetricTypes
+	// Stackdriver requires metrics to be prefixed with a common prefix.
+	// This prefix changes based on the services the metrics belongs to.
+	ServiceMetricPrefix string   `config:"service_metric_prefix"`
+	MetricTypes         []string `config:"metric_types" validate:"required"`
+	Aligner             string   `config:"aligner"`
+}
+
+func (mc metricsConfig) AddPrefixTo(metric string) string {
+	prefix := mc.ServiceMetricPrefix
+	// NOTE: fallback to Google Cloud prefix for backward compatibility
+	// Prefix <service>.googleapis.com/ works only for Google Cloud metrics
+	// List: https://cloud.google.com/monitoring/api/metrics_gcp
+	if prefix == "" {
+		prefix = mc.ServiceName + ".googleapis.com"
+	}
+
+	return path.Join(prefix, metric)
 }
 
 type metricMeta struct {
@@ -228,7 +245,7 @@ func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.Met
 
 	for _, sdc := range m.MetricsConfig {
 		for _, mt := range sdc.MetricTypes {
-			req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.ServiceName+".googleapis.com/"+mt)
+			req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.AddPrefixTo(mt))
 			it := client.ListMetricDescriptors(ctx, req)
 
 			for {

x-pack/metricbeat/module/gcp/metrics/metricset_test.go

@@ -0,0 +1,45 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package metrics
+
+import "testing"
+
+var fakeMetricsConfig = []metricsConfig{
+	{"billing", "", []string{}, ""},
+	{"billing", "foobar/", []string{}, ""},
+	{"billing", "foobar", []string{}, ""},
+}
+
+func Test_metricsConfig_AddPrefixTo(t *testing.T) {
+	metric := "awesome/metric"
+	tests := []struct {
+		name   string
+		fields metricsConfig
+		want   string
+	}{
+		{
+			name:   "only service name",
+			fields: fakeMetricsConfig[0],
+			want:   "billing.googleapis.com/" + metric,
+		},
+		{
+			name:   "service metric prefix override",
+			fields: fakeMetricsConfig[1],
+			want:   "foobar/" + metric,
+		},
+		{
+			name:   "service metric prefix override (without trailing /)",
+			fields: fakeMetricsConfig[2],
+			want:   "foobar/" + metric,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.fields.AddPrefixTo(metric); got != tt.want {
+				t.Errorf("metricsConfig.AddPrefixTo(%s) = %v, want %v", metric, got, tt.want)
+			}
+		})
+	}
+}

x-pack/metricbeat/modules.d/gcp.yml.disabled

@@ -44,6 +44,20 @@
         - "instance/cpu/utilization"
         - "instance/uptime"
 
+- module: gcp
+  metricsets:
+    - metrics
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  exclude_labels: false
+  period: 1m
+  metrics:
+    - aligner: ALIGN_NONE
+      service: kubernetes
+      service_metric_prefix: kubernetes.io/
+      metric_types:
+        - "container/cpu/core_usage_time"
+
 - module: gcp
   metricsets:
     - billing