From 2c94692ed617f7d5e3b7566eed1bcecfd68ad3b8 Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 13:30:09 +0300 Subject: [PATCH 01/12] add cloudsql metadata --- .../module/gcp/metrics/cloudsql/metadata.go | 213 ++++++++++++++++++ .../module/gcp/metrics/metadata_services.go | 3 + 2 files changed, 216 insertions(+) create mode 100644 x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go new file mode 100644 index 00000000000..39176a88c9f --- /dev/null +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -0,0 +1,213 @@ +package cloudsql + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "google.golang.org/api/option" + "google.golang.org/api/sqladmin/v1" + monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" +) + +const ( + cacheTTL = 30 * time.Second + initialCacheSize = 13 +) + +// NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. +func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { + return &metadataCollector{ + projectID: projectID, + zone: zone, + region: region, + regions: regions, + opt: opt, + instanceCache: common.NewCache(cacheTTL, initialCacheSize), + logger: logp.NewLogger("metrics-cloudsql"), + }, nil +} + +// cloudsqlMetadata is an object to store data in between the extraction and the writing in the destination (to uncouple +// reading and writing in the same method) +type cloudsqlMetadata struct { + // projectID string + region string + instanceID string + machineType string + databaseVersion string + + // ts *monitoringpb.TimeSeries + + User map[string]string + Metadata map[string]string + Metrics interface{} + System interface{} +} + +type metadataCollector struct { + projectID string + zone string + region string + regions []string + opt []option.ClientOption + instanceCache *common.Cache + logger *logp.Logger +} + +func getDatabaseNameAndVersion(db string) mapstr.M { + parts := strings.SplitN(strings.ToLower(db), "_", 2) + + var cloudsqlDb mapstr.M + + switch { + case db == "SQL_DATABASE_VERSION_UNSPECIFIED": + cloudsqlDb = mapstr.M{ + "name": "sql", + "version": "unspecified", + } + case strings.Contains(parts[0], "sqlserver"): + cloudsqlDb = mapstr.M{ + "name": strings.ToLower(parts[0]), + "version": strings.ToLower(parts[1]), + } + default: + version := strings.ReplaceAll(parts[1], "_", ".") + cloudsqlDb = mapstr.M{ + "name": strings.ToLower(parts[0]), + "version": version, + } + } + + return cloudsqlDb +} + +// Metadata implements googlecloud.MetadataCollector to the known set of labels from a CloudSQL TimeSeries single point of data. +func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.TimeSeries) (gcp.MetadataCollectorData, error) { + cloudsqlMetadata, err := s.instanceMetadata(ctx, s.instanceID(resp), s.instanceRegion(resp)) + if err != nil { + return gcp.MetadataCollectorData{}, err + } + + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + + metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) + if err != nil { + return gcp.MetadataCollectorData{}, err + } + + if cloudsqlMetadata.machineType != "" { + lastIndex := strings.LastIndex(cloudsqlMetadata.machineType, "/") + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudMachineTypeKey, cloudsqlMetadata.machineType[lastIndex+1:]) + } + + cloudsqlMetadata.Metrics = metadataCollectorData.Labels[gcp.LabelMetrics] + cloudsqlMetadata.System = metadataCollectorData.Labels[gcp.LabelSystem] + + if cloudsqlMetadata.databaseVersion != "" { + err := mapstr.MergeFields(metadataCollectorData.Labels, mapstr.M{ + "cloudsql": getDatabaseNameAndVersion(cloudsqlMetadata.databaseVersion), + }, true) + if err != nil { + s.logger.Warnf("failed merging cloudsql to label fields: %w", err) + } + } + + return metadataCollectorData, nil +} + +func (s *metadataCollector) instanceID(ts *monitoringpb.TimeSeries) string { + if ts.Resource != nil && ts.Resource.Labels != nil { + return ts.Resource.Labels["database_id"] + } + + return "" +} + +func (s *metadataCollector) instanceRegion(ts *monitoringpb.TimeSeries) string { + if ts.Resource != nil && ts.Resource.Labels != nil { + return ts.Resource.Labels["region"] + } + + return "" +} + +// instanceMetadata returns the labels of an instance +func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, region string) (*cloudsqlMetadata, error) { + instance, err := s.instance(ctx, instanceID) + if err != nil { + return nil, fmt.Errorf("error trying to get data from instance '%s' %w", instanceID, err) + } + + cloudsqlMetadata := &cloudsqlMetadata{ + instanceID: instanceID, + region: region, + } + + if instance == nil { + return cloudsqlMetadata, nil + } + + if instance.DatabaseVersion != "" { + cloudsqlMetadata.databaseVersion = instance.DatabaseVersion + } + + return cloudsqlMetadata, nil +} + +func (s *metadataCollector) refreshInstanceCache(ctx context.Context) { + // only refresh cache if it is empty + if s.instanceCache.Size() > 0 { + return + } + + s.logger.Debugf("refresh cache with Instances.List API") + + service, _ := sqladmin.NewService(ctx, s.opt...) + + req := service.Instances.List(s.projectID) + if err := req.Pages(ctx, func(page *sqladmin.InstancesListResponse) error { + for _, instancesScopedList := range page.Items { + s.instanceCache.Put(fmt.Sprintf("%s:%s", instancesScopedList.Project, instancesScopedList.Name), instancesScopedList) + } + return nil + }); err != nil { + s.logger.Errorf("cloudsql Instances.List error: %v", err) + } +} + +func (s *metadataCollector) instance(ctx context.Context, instanceName string) (*sqladmin.DatabaseInstance, error) { + s.refreshInstanceCache(ctx) + instanceCachedData := s.instanceCache.Get(instanceName) + if instanceCachedData != nil { + if cloudsqlInstance, ok := instanceCachedData.(*sqladmin.DatabaseInstance); ok { + return cloudsqlInstance, nil + } + } + + return nil, nil +} + +func (s *metadataCollector) ID(ctx context.Context, in *gcp.MetadataCollectorInputData) (string, error) { + metadata, err := s.Metadata(ctx, in.TimeSeries) + if err != nil { + return "", err + } + + metadata.ECS.Update(metadata.Labels) + if in.Timestamp != nil { + metadata.ECS.Put("timestamp", in.Timestamp) + } else if in.Point != nil { + metadata.ECS.Put("timestamp", in.Point.Interval.EndTime) + } else { + return "", errors.New("no timestamp information found") + } + + return metadata.ECS.String(), nil +} diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 2f3f9a3f811..40ab1dcb788 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -6,6 +6,7 @@ package metrics import ( "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics/cloudsql" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics/compute" ) @@ -15,6 +16,8 @@ func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataServ switch serviceName { case gcp.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + case "cloudsql": + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) default: return nil, nil } From 53eff0377d9be2cda2e8f6aaccfe873b2b2f7e3c Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 13:30:20 +0300 Subject: [PATCH 02/12] add cloudsql metadata tests --- .../gcp/metrics/cloudsql/metadata_test.go | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go new file mode 100644 index 00000000000..fc2af3d3357 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -0,0 +1,222 @@ +package cloudsql + +import ( + "testing" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/golang/protobuf/ptypes/timestamp" + "google.golang.org/genproto/googleapis/api/metric" + "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/genproto/googleapis/monitoring/v3" + "gotest.tools/assert" +) + +var fake = &monitoring.TimeSeries{ + Resource: &monitoredres.MonitoredResource{ + Type: "gce_instance", + Labels: map[string]string{ + "database_id": "db", + "project_id": "elastic-metricbeat", + "region": "us-central1", + }, + }, + Metadata: &monitoredres.MonitoredResourceMetadata{ + UserLabels: map[string]string{ + "user": "label", + }, + }, + Metric: &metric.Metric{ + Labels: map[string]string{ + "instance_name": "instance-1", + }, + Type: "compute.googleapis.com/instance/cpu/usage_time", + }, + MetricKind: metric.MetricDescriptor_GAUGE, + ValueType: metric.MetricDescriptor_DOUBLE, + Points: []*monitoring.Point{{ + Value: &monitoring.TypedValue{ + Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, + }, + Interval: &monitoring.TimeInterval{ + StartTime: ×tamp.Timestamp{ + Seconds: 1569932700, + }, + EndTime: ×tamp.Timestamp{ + Seconds: 1569932700, + }, + }, + }, { + Value: &monitoring.TypedValue{ + Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, + }, + Interval: &monitoring.TimeInterval{ + StartTime: ×tamp.Timestamp{ + Seconds: 1569932640, + }, + EndTime: ×tamp.Timestamp{ + Seconds: 1569932640, + }, + }, + }}, +} + +var m = &metadataCollector{ + projectID: "projectID", + instanceCache: common.NewCache(30*time.Second, 13), +} + +func TestInstanceID(t *testing.T) { + instanceID := m.instanceID(fake) + assert.Equal(t, "db", instanceID) +} + +func TestInstanceRegion(t *testing.T) { + zone := m.instanceRegion(fake) + assert.Equal(t, "us-central1", zone) +} + +func TestGetDatabaseNameAndVersion(t *testing.T) { + cases := []struct { + name string + db string + expected mapstr.M + }{ + { + name: "sql unspecified", + db: "SQL_DATABASE_VERSION_UNSPECIFIED", + expected: mapstr.M{ + "name": "sql", + "version": "unspecified", + }, + }, + { + name: "mysql 5.1", + db: "MYSQL_5_1", + expected: mapstr.M{ + "name": "mysql", + "version": "5.1", + }, + }, + { + name: "mysql 5.5", + db: "MYSQL_5_5", + expected: mapstr.M{ + "name": "mysql", + "version": "5.5", + }, + }, + { + name: "mysql 5.6", + db: "MYSQL_5_6", + expected: mapstr.M{ + "name": "mysql", + "version": "5.6", + }, + }, + { + name: "mysql 5.7", + db: "MYSQL_5_7", + expected: mapstr.M{ + "name": "mysql", + "version": "5.7", + }, + }, + { + name: "postgres 9.6", + db: "POSTGRES_9_6", + expected: mapstr.M{ + "name": "postgres", + "version": "9.6", + }, + }, + { + name: "postgres 11", + db: "POSTGRES_11", + expected: mapstr.M{ + "name": "postgres", + "version": "11", + }, + }, + { + name: "SQLSERVER_2017_STANDARD", + db: "SQLSERVER_2017_STANDARD", + expected: mapstr.M{ + "name": "sqlserver", + "version": "2017_standard", + }, + }, + { + name: "SQLSERVER_2017_ENTERPRISE", + db: "SQLSERVER_2017_ENTERPRISE", + expected: mapstr.M{ + "name": "sqlserver", + "version": "2017_enterprise", + }, + }, + { + name: "SQLSERVER_2017_EXPRESS", + db: "SQLSERVER_2017_EXPRESS", + expected: mapstr.M{ + "name": "sqlserver", + "version": "2017_express", + }, + }, + { + name: "SQLSERVER_2017_WEB", + db: "SQLSERVER_2017_WEB", + expected: mapstr.M{ + "name": "sqlserver", + "version": "2017_web", + }, + }, + { + name: "POSTGRES_10", + db: "POSTGRES_10", + expected: mapstr.M{ + "name": "postgres", + "version": "10", + }, + }, + { + name: "POSTGRES_12", + db: "POSTGRES_12", + expected: mapstr.M{ + "name": "postgres", + "version": "12", + }, + }, + { + name: "MYSQL_8_0", + db: "MYSQL_8_0", + expected: mapstr.M{ + "name": "mysql", + "version": "8.0", + }, + }, + { + name: "MYSQL_8_0_18", + db: "MYSQL_8_0_18", + expected: mapstr.M{ + "name": "mysql", + "version": "8.0.18", + }, + }, + { + name: "SQLSERVER_2019_STANDARD", + db: "SQLSERVER_2019_STANDARD", + expected: mapstr.M{ + "name": "sqlserver", + "version": "2019_standard", + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + db := getDatabaseNameAndVersion(c.db) + assert.DeepEqual(t, db, c.expected) + }) + } +} From 86bcf673d68b36e459c6a33bf64feafac30ead97 Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 13:54:35 +0300 Subject: [PATCH 03/12] add cloudsql service constant --- x-pack/metricbeat/module/gcp/constants.go | 1 + x-pack/metricbeat/module/gcp/metrics/metadata_services.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index cfdd96943fe..fa6bc4fb735 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -24,6 +24,7 @@ const ( ServiceStorage = "storage" ServiceFirestore = "firestore" ServiceDataproc = "dataproc" + ServiceCloudSQL = "cloudsql" ) //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 40ab1dcb788..95e7172f211 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,7 +16,7 @@ func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataServ switch serviceName { case gcp.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) - case "cloudsql": + case gcp.ServiceCloudSQL: return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) default: return nil, nil From 455a83e26ded64b02e40867f5a2ce1735bebffb2 Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 14:00:24 +0300 Subject: [PATCH 04/12] add license headers --- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go | 4 ++++ .../metricbeat/module/gcp/metrics/cloudsql/metadata_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 39176a88c9f..6d994e8b90c 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package cloudsql import ( diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index fc2af3d3357..9788b62ed57 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package cloudsql import ( From cac3ba3a6110bdb22ca18292c10716c707df9133 Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 15:15:05 +0300 Subject: [PATCH 05/12] mage update --- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go | 7 ++++--- .../module/gcp/metrics/cloudsql/metadata_test.go | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 6d994e8b90c..d3a1a285656 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -11,13 +11,14 @@ import ( "strings" "time" + "google.golang.org/api/option" + "google.golang.org/api/sqladmin/v1" + monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "google.golang.org/api/option" - "google.golang.org/api/sqladmin/v1" - monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" ) const ( diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index 9788b62ed57..3b748d30b9a 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -8,13 +8,14 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-libs/mapstr" "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" "google.golang.org/genproto/googleapis/monitoring/v3" "gotest.tools/assert" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/elastic-agent-libs/mapstr" ) var fake = &monitoring.TimeSeries{ From 365bf59bb08a8924092e3a73f90b9a0503a73f9f Mon Sep 17 00:00:00 2001 From: gab Date: Tue, 13 Sep 2022 15:20:17 +0300 Subject: [PATCH 06/12] fix golangci-lint --- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index d3a1a285656..a0d28b4540b 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -207,9 +207,9 @@ func (s *metadataCollector) ID(ctx context.Context, in *gcp.MetadataCollectorInp metadata.ECS.Update(metadata.Labels) if in.Timestamp != nil { - metadata.ECS.Put("timestamp", in.Timestamp) + _, _ = metadata.ECS.Put("timestamp", in.Timestamp) } else if in.Point != nil { - metadata.ECS.Put("timestamp", in.Point.Interval.EndTime) + _, _ = metadata.ECS.Put("timestamp", in.Point.Interval.EndTime) } else { return "", errors.New("no timestamp information found") } From 289ad7dfd861e5db337dd09c6c6ea39f37400443 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 13:29:40 +0200 Subject: [PATCH 07/12] remove cache --- .../module/gcp/metrics/cloudsql/metadata.go | 124 +++++++++--------- .../gcp/metrics/cloudsql/metadata_test.go | 5 +- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index a0d28b4540b..f384f846e92 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -9,33 +9,26 @@ import ( "errors" "fmt" "strings" - "time" "google.golang.org/api/option" "google.golang.org/api/sqladmin/v1" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) -const ( - cacheTTL = 30 * time.Second - initialCacheSize = 13 -) - // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instanceCache: common.NewCache(cacheTTL, initialCacheSize), - logger: logp.NewLogger("metrics-cloudsql"), + projectID: projectID, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*sqladmin.DatabaseInstance), + logger: logp.NewLogger("metrics-cloudsql"), }, nil } @@ -43,9 +36,9 @@ func NewMetadataService(projectID, zone string, region string, regions []string, // reading and writing in the same method) type cloudsqlMetadata struct { // projectID string - region string - instanceID string - machineType string + region string + instanceID string + // machineType string databaseVersion string // ts *monitoringpb.TimeSeries @@ -57,13 +50,13 @@ type cloudsqlMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption - instanceCache *common.Cache - logger *logp.Logger + projectID string + zone string + region string + regions []string + opt []option.ClientOption + instances map[string]*sqladmin.DatabaseInstance + logger *logp.Logger } func getDatabaseNameAndVersion(db string) mapstr.M { @@ -107,9 +100,12 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - if cloudsqlMetadata.machineType != "" { - lastIndex := strings.LastIndex(cloudsqlMetadata.machineType, "/") - _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudMachineTypeKey, cloudsqlMetadata.machineType[lastIndex+1:]) + if resp.Resource != nil && resp.Resource.Labels != nil { + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, resp.Resource.Labels[gcp.TimeSeriesResponsePathForECSInstanceID]) + } + + if resp.Metric.Labels != nil { + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, resp.Metric.Labels[gcp.TimeSeriesResponsePathForECSInstanceName]) } cloudsqlMetadata.Metrics = metadataCollectorData.Labels[gcp.LabelMetrics] @@ -120,7 +116,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim "cloudsql": getDatabaseNameAndVersion(cloudsqlMetadata.databaseVersion), }, true) if err != nil { - s.logger.Warnf("failed merging cloudsql to label fields: %w", err) + s.logger.Warnf("failed merging cloudsql to label fields: %v", err) } } @@ -156,6 +152,7 @@ func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, re } if instance == nil { + s.logger.Debugf("couldn't find instance %s, call sqladmin Instances.List", instanceID) return cloudsqlMetadata, nil } @@ -166,39 +163,6 @@ func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, re return cloudsqlMetadata, nil } -func (s *metadataCollector) refreshInstanceCache(ctx context.Context) { - // only refresh cache if it is empty - if s.instanceCache.Size() > 0 { - return - } - - s.logger.Debugf("refresh cache with Instances.List API") - - service, _ := sqladmin.NewService(ctx, s.opt...) - - req := service.Instances.List(s.projectID) - if err := req.Pages(ctx, func(page *sqladmin.InstancesListResponse) error { - for _, instancesScopedList := range page.Items { - s.instanceCache.Put(fmt.Sprintf("%s:%s", instancesScopedList.Project, instancesScopedList.Name), instancesScopedList) - } - return nil - }); err != nil { - s.logger.Errorf("cloudsql Instances.List error: %v", err) - } -} - -func (s *metadataCollector) instance(ctx context.Context, instanceName string) (*sqladmin.DatabaseInstance, error) { - s.refreshInstanceCache(ctx) - instanceCachedData := s.instanceCache.Get(instanceName) - if instanceCachedData != nil { - if cloudsqlInstance, ok := instanceCachedData.(*sqladmin.DatabaseInstance); ok { - return cloudsqlInstance, nil - } - } - - return nil, nil -} - func (s *metadataCollector) ID(ctx context.Context, in *gcp.MetadataCollectorInputData) (string, error) { metadata, err := s.Metadata(ctx, in.TimeSeries) if err != nil { @@ -216,3 +180,41 @@ func (s *metadataCollector) ID(ctx context.Context, in *gcp.MetadataCollectorInp return metadata.ECS.String(), nil } + +func (s *metadataCollector) instance(ctx context.Context, instanceName string) (*sqladmin.DatabaseInstance, error) { + s.getInstances(ctx) + + instance, ok := s.instances[instanceName] + if ok { + return instance, nil + } + + // Remake the compute instances map to avoid having stale data. + s.instances = make(map[string]*sqladmin.DatabaseInstance) + + return nil, nil +} + +func (s *metadataCollector) getInstances(ctx context.Context) { + if len(s.instances) > 0 { + return + } + + s.logger.Debug("sqladmin Instances.List API") + + service, err := sqladmin.NewService(ctx, s.opt...) + if err != nil { + s.logger.Errorf("error getting client from sqladmin service: %v", err) + return + } + + req := service.Instances.List(s.projectID) + if err := req.Pages(ctx, func(page *sqladmin.InstancesListResponse) error { + for _, instancesScopedList := range page.Items { + s.instances[fmt.Sprintf("%s:%s", instancesScopedList.Project, instancesScopedList.Name)] = instancesScopedList + } + return nil + }); err != nil { + s.logger.Errorf("sqladmin Instances.List error: %v", err) + } +} diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index 3b748d30b9a..2f60d294bbb 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -6,7 +6,6 @@ package cloudsql import ( "testing" - "time" "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/genproto/googleapis/api/metric" @@ -14,7 +13,6 @@ import ( "google.golang.org/genproto/googleapis/monitoring/v3" "gotest.tools/assert" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -68,8 +66,7 @@ var fake = &monitoring.TimeSeries{ } var m = &metadataCollector{ - projectID: "projectID", - instanceCache: common.NewCache(30*time.Second, 13), + projectID: "projectID", } func TestInstanceID(t *testing.T) { From 902574d6367073593d1e74b181940f025c7defdb Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 14:38:55 +0200 Subject: [PATCH 08/12] add instance name field --- .../module/gcp/metrics/cloudsql/metadata.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index f384f846e92..e7d9d648e13 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -36,8 +36,9 @@ func NewMetadataService(projectID, zone string, region string, regions []string, // reading and writing in the same method) type cloudsqlMetadata struct { // projectID string - region string - instanceID string + region string + instanceID string + instanceName string // machineType string databaseVersion string @@ -101,11 +102,11 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim } if resp.Resource != nil && resp.Resource.Labels != nil { - _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, resp.Resource.Labels[gcp.TimeSeriesResponsePathForECSInstanceID]) + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, cloudsqlMetadata.instanceID) } if resp.Metric.Labels != nil { - _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, resp.Metric.Labels[gcp.TimeSeriesResponsePathForECSInstanceName]) + _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, cloudsqlMetadata.instanceName) } cloudsqlMetadata.Metrics = metadataCollectorData.Labels[gcp.LabelMetrics] @@ -160,6 +161,10 @@ func (s *metadataCollector) instanceMetadata(ctx context.Context, instanceID, re cloudsqlMetadata.databaseVersion = instance.DatabaseVersion } + if instance.Name != "" { + cloudsqlMetadata.instanceName = instance.Name + } + return cloudsqlMetadata, nil } From e5fcfb310857e8e46aae3af49d5148cbc8156e58 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 16:56:10 +0200 Subject: [PATCH 09/12] add instances comment --- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index e7d9d648e13..a3c9f447779 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -56,6 +56,8 @@ type metadataCollector struct { region string regions []string opt []option.ClientOption + // NOTE: instances holds data used for all metrics collected in a given period + // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*sqladmin.DatabaseInstance logger *logp.Logger } From 3775cfd6a792456bb7e53b114dc48a2b88cdb0b3 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 18:24:17 +0200 Subject: [PATCH 10/12] add import aliases --- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go | 2 +- x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index a3c9f447779..66a1d30f197 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -11,7 +11,7 @@ import ( "strings" "google.golang.org/api/option" - "google.golang.org/api/sqladmin/v1" + sqladmin "google.golang.org/api/sqladmin/v1" monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index 2f60d294bbb..3f54e8aae52 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -10,7 +10,7 @@ import ( "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" - "google.golang.org/genproto/googleapis/monitoring/v3" + monitoring "google.golang.org/genproto/googleapis/monitoring/v3" "gotest.tools/assert" "github.com/elastic/elastic-agent-libs/mapstr" From 13d6d9732bc62d95e6f803260172344ca0318ced Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 18:35:40 +0200 Subject: [PATCH 11/12] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7409da2e4ec..46c084c1fd0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -177,6 +177,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - Add Data Granularity option to AWS module to allow for for fewer API calls of longer periods and keep small intervals. {issue}33133[33133] {pull}33166[33166] - Update README file on how to run Metricbeat on Kubernetes. {pull}33308[33308] - Add per-thread metrics to system_summary {pull}33614[33614] +- Add GCP CloudSQL metadata {pull}33066[33066] *Packetbeat* From ce3f69d8f9e9d4cd06510f50afcdc4cf3ff830bf Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 15 Nov 2022 18:45:02 +0200 Subject: [PATCH 12/12] remove cloud instance and unused metadata fields --- .../module/gcp/metrics/cloudsql/metadata.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 66a1d30f197..53fa78dc249 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -35,15 +35,11 @@ func NewMetadataService(projectID, zone string, region string, regions []string, // cloudsqlMetadata is an object to store data in between the extraction and the writing in the destination (to uncouple // reading and writing in the same method) type cloudsqlMetadata struct { - // projectID string - region string - instanceID string - instanceName string - // machineType string + region string + instanceID string + instanceName string databaseVersion string - // ts *monitoringpb.TimeSeries - User map[string]string Metadata map[string]string Metrics interface{} @@ -103,14 +99,6 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - if resp.Resource != nil && resp.Resource.Labels != nil { - _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceIDKey, cloudsqlMetadata.instanceID) - } - - if resp.Metric.Labels != nil { - _, _ = metadataCollectorData.ECS.Put(gcp.ECSCloudInstanceNameKey, cloudsqlMetadata.instanceName) - } - cloudsqlMetadata.Metrics = metadataCollectorData.Labels[gcp.LabelMetrics] cloudsqlMetadata.System = metadataCollectorData.Labels[gcp.LabelSystem]