diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 64ef7c5d2e5..b6a97881c73 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -512,6 +512,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix memory leak in SQL module when database is not available. {issue}25840[25840] {pull}26607[26607] - Fix aws metric tags with resourcegroupstaggingapi paginator. {issue}26385[26385] {pull}26443[26443] - Fix quoting in GCP billing table name {issue}26855[26855] {pull}26870[26870] +- Allow metric prefix override per service in gcp module. {pull}26960[26960] *Packetbeat* diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index ded907e5bf9..fbca8e75cc0 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -286,6 +286,20 @@ metricbeat.modules: - "instance/cpu/utilization" - "instance/uptime" +- module: gcp + metricsets: + - metrics + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: kubernetes + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" + - module: gcp metricsets: - billing diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1b5fc6ab8b9..65f84166d72 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -569,6 +569,20 @@ metricbeat.modules: - "instance/cpu/utilization" - "instance/uptime" +- module: gcp + metricsets: + - metrics + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: kubernetes + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" + - module: gcp metricsets: - billing diff --git a/x-pack/metricbeat/module/gcp/_meta/config.yml b/x-pack/metricbeat/module/gcp/_meta/config.yml index 9cf4f8387df..f29075b4ca8 100644 --- a/x-pack/metricbeat/module/gcp/_meta/config.yml +++ b/x-pack/metricbeat/module/gcp/_meta/config.yml @@ -41,6 +41,20 @@ - "instance/cpu/utilization" - "instance/uptime" +- module: gcp + metricsets: + - metrics + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: kubernetes + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" + - module: gcp metricsets: - billing diff --git a/x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc index 981c1675095..83fb4e1c9c0 100644 --- a/x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/metrics/_meta/docs.asciidoc @@ -16,12 +16,6 @@ call. [float] == Metricset config and parameters -* *metric_types*: Required, a list of metric type strings, or a list of metric -type prefixes. For example, `instance/cpu` is the prefix for metric type -`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the -`ListTimeSeries` API can return any number of time series from a single metric -type. Metric type is to used for identifying a specific time series. - * *aligner*: A single string with which aggregation operation need to be applied onto time series data for ListTimeSeries API. If it's not given, default aligner is `ALIGN_NONE`. Google Cloud also supports `ALIGN_DELTA`, `ALIGN_RATE`, @@ -30,6 +24,27 @@ Please see https://cloud.google.com/monitoring/api/ref_v3/rpc/google.monitoring.v3#aligner[Aggregation Aligner] for the full list of aligners. +* *metric_types*: Required, a list of metric type strings, or a list of metric +type prefixes. For example, `instance/cpu` is the prefix for metric type +`instance/cpu/usage_time`, `instance/cpu/utilization` etc Each call of the +`ListTimeSeries` API can return any number of time series from a single metric +type. Metric type is to used for identifying a specific time series. + +* *service*: Required, the name of the service for related metrics. This should +be a valid Google Cloud service name. Service names may be viewed from the +corresponding page from https://cloud.google.com/monitoring/api/metrics[GCP Metrics list documentation]. +The `service` field is used to compute the GCP Metric prefix, unless +`service_metric_prefix` is set. + +* *service_metric_prefix*: A string containing the full Metric prefix as +specified in the GCP documentation. +All metrics from GCP Monitoring API require a prefix. When +`service_metric_prefix` is empty, the prefix default to a value computed using +the `service` value: `.googleapis.com/`. This default works for any +services under "Google Cloud metrics", but does not work for other services +(`kubernetes` aka GKE for example). +This option allow to override the default and specify an arbitrary metric prefix. + [float] === Example Configuration * `metrics` metricset is enabled to collect metrics from all zones under @@ -124,3 +139,27 @@ every minute with no aggregation. The metric types in `compute` service with metric_types: - "instance/cpu" ---- + +* `metrics` metricset is enabled to collect metrics from all zones under +`europe-west1-c` region in `elastic-observability` project. One set of metrics +will be collected: core usage time for containers in GCP GKE. +Note that the is required to use `service_metric_prefix` to override the default +metric prefix, as for GKE metrics the required prefix is `kubernetes.io/` + ++ +[source,yaml] +---- +- module: googlecloud + metricsets: + - metrics + zone: "europe-west1-c" + project_id: elastic-observability + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - service: kubernetes + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" +---- diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index f8255321a12..7a59434c09b 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -7,6 +7,7 @@ package metrics import ( "context" "fmt" + "path" "time" monitoring "cloud.google.com/go/monitoring/apiv3" @@ -51,9 +52,25 @@ type MetricSet struct { //metricsConfig holds a configuration specific for metrics metricset. type metricsConfig struct { - ServiceName string `config:"service" validate:"required"` - MetricTypes []string `config:"metric_types" validate:"required"` - Aligner string `config:"aligner"` + ServiceName string `config:"service" validate:"required"` + // MetricPrefix allows to specify the prefix string for MetricTypes + // Stackdriver requires metrics to be prefixed with a common prefix. + // This prefix changes based on the services the metrics belongs to. + ServiceMetricPrefix string `config:"service_metric_prefix"` + MetricTypes []string `config:"metric_types" validate:"required"` + Aligner string `config:"aligner"` +} + +func (mc metricsConfig) AddPrefixTo(metric string) string { + prefix := mc.ServiceMetricPrefix + // NOTE: fallback to Google Cloud prefix for backward compatibility + // Prefix .googleapis.com/ works only for Google Cloud metrics + // List: https://cloud.google.com/monitoring/api/metrics_gcp + if prefix == "" { + prefix = mc.ServiceName + ".googleapis.com" + } + + return path.Join(prefix, metric) } type metricMeta struct { @@ -228,7 +245,7 @@ func (m *MetricSet) metricDescriptor(ctx context.Context, client *monitoring.Met for _, sdc := range m.MetricsConfig { for _, mt := range sdc.MetricTypes { - req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.ServiceName+".googleapis.com/"+mt) + req.Filter = fmt.Sprintf(`metric.type = starts_with("%s")`, sdc.AddPrefixTo(mt)) it := client.ListMetricDescriptors(ctx, req) for { diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset_test.go b/x-pack/metricbeat/module/gcp/metrics/metricset_test.go new file mode 100644 index 00000000000..e7f0418f27f --- /dev/null +++ b/x-pack/metricbeat/module/gcp/metrics/metricset_test.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package metrics + +import "testing" + +var fakeMetricsConfig = []metricsConfig{ + {"billing", "", []string{}, ""}, + {"billing", "foobar/", []string{}, ""}, + {"billing", "foobar", []string{}, ""}, +} + +func Test_metricsConfig_AddPrefixTo(t *testing.T) { + metric := "awesome/metric" + tests := []struct { + name string + fields metricsConfig + want string + }{ + { + name: "only service name", + fields: fakeMetricsConfig[0], + want: "billing.googleapis.com/" + metric, + }, + { + name: "service metric prefix override", + fields: fakeMetricsConfig[1], + want: "foobar/" + metric, + }, + { + name: "service metric prefix override (without trailing /)", + fields: fakeMetricsConfig[2], + want: "foobar/" + metric, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.fields.AddPrefixTo(metric); got != tt.want { + t.Errorf("metricsConfig.AddPrefixTo(%s) = %v, want %v", metric, got, tt.want) + } + }) + } +} diff --git a/x-pack/metricbeat/modules.d/gcp.yml.disabled b/x-pack/metricbeat/modules.d/gcp.yml.disabled index 1d5c7e6a69d..38428f335fc 100644 --- a/x-pack/metricbeat/modules.d/gcp.yml.disabled +++ b/x-pack/metricbeat/modules.d/gcp.yml.disabled @@ -44,6 +44,20 @@ - "instance/cpu/utilization" - "instance/uptime" +- module: gcp + metricsets: + - metrics + project_id: "your project id" + credentials_file_path: "your JSON credentials file path" + exclude_labels: false + period: 1m + metrics: + - aligner: ALIGN_NONE + service: kubernetes + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" + - module: gcp metricsets: - billing