From 95a0a10b836769fb84e4eb0423526c7a55ec10fe Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 20 May 2020 06:56:51 -0600 Subject: [PATCH] [Metricbeat] Rename tags to tags_filter for cloudwatch metricset (#16733) * Deprecate tags config parameter to use tags_filter instead for cloudwatch metricset (cherry picked from commit 7e5e98cf266ecd71ec4ce32da7f5d0d1e25342a3) --- CHANGELOG.next.asciidoc | 2 +- .../module/aws/cloudwatch/cloudwatch.go | 21 +++- .../module/aws/cloudwatch/cloudwatch_test.go | 118 ++++++++++++------ 3 files changed, 98 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c9ef1985c559..be976665fb1b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -535,7 +535,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* *Metricbeat* - +- Deprecate tags config parameter in cloudwatch metricset. {pull}16733[16733] *Packetbeat* diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 4a38a3fc5bf9..2ec9fe5461e1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" @@ -69,7 +70,15 @@ type Config struct { Dimensions []Dimension `config:"dimensions"` ResourceTypeFilter string `config:"tags.resource_type_filter"` Statistic []string `config:"statistic"` - Tags []aws.Tag `config:"tags"` + Tags []aws.Tag `config:"tags"` // Deprecated. +} + +// Validate checks for deprecated config options +func (c Config) Validate() error { + if c.Tags != nil { + cfgwarn.Deprecate("8.0.0", "tags is deprecated. Use tags_filter instead") + } + return nil } type metricsWithStatistics struct { @@ -292,8 +301,9 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n for _, config := range m.CloudwatchConfigs { // If tags_filter on metricset level is given, overwrite tags in // cloudwatch metrics with tags_filter. + tagsFilter := config.Tags if m.MetricSet.TagsFilter != nil { - config.Tags = m.MetricSet.TagsFilter + tagsFilter = m.MetricSet.TagsFilter } // If there is no statistic method specified, then use the default. @@ -327,10 +337,9 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n if config.ResourceTypeFilter != "" { if _, ok := resourceTypesWithTags[config.ResourceTypeFilter]; ok { - resourceTypesWithTags[config.ResourceTypeFilter] = config.Tags - + resourceTypesWithTags[config.ResourceTypeFilter] = tagsFilter } else { - resourceTypesWithTags[config.ResourceTypeFilter] = append(resourceTypesWithTags[config.ResourceTypeFilter], config.Tags...) + resourceTypesWithTags[config.ResourceTypeFilter] = append(resourceTypesWithTags[config.ResourceTypeFilter], tagsFilter...) } } continue @@ -338,7 +347,7 @@ func (m *MetricSet) readCloudwatchConfig() (listMetricWithDetail, map[string][]n configPerNamespace := namespaceDetail{ names: config.MetricName, - tags: config.Tags, + tags: tagsFilter, statistics: config.Statistic, resourceTypeFilter: config.ResourceTypeFilter, dimensions: cloudwatchDimensions, diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index f3e2fb9f38d2..e6b5cdebe5a0 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -213,6 +213,53 @@ func TestReadCloudwatchConfig(t *testing.T) { resourceTypeFilters: resourceTypeFiltersEC2RDS, } + resourceTypeFiltersEC2RDSWithTag := map[string][]aws.Tag{} + resourceTypeFiltersEC2RDSWithTag["ec2:instance"] = []aws.Tag{ + { + Key: "name", + Value: "test", + }, + } + resourceTypeFiltersEC2RDSWithTag["rds"] = []aws.Tag{ + { + Key: "name", + Value: "test", + }, + } + expectedListMetricWithDetailEC2RDSWithTag := listMetricWithDetail{ + metricsWithStats: []metricsWithStatistics{ + { + cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: awssdk.String("InstanceId"), + Value: awssdk.String("i-1"), + }}, + MetricName: awssdk.String("CPUUtilization"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + nil, + }, + { + cloudwatch.Metric{ + Dimensions: []cloudwatch.Dimension{{ + Name: awssdk.String("DBClusterIdentifier"), + Value: awssdk.String("test1-cluster"), + }, + { + Name: awssdk.String("Role"), + Value: awssdk.String("READER"), + }}, + MetricName: awssdk.String("CommitThroughput"), + Namespace: awssdk.String("AWS/RDS"), + }, + []string{"Average"}, + nil, + }, + }, + resourceTypeFilters: resourceTypeFiltersEC2RDSWithTag, + } + expectedNamespaceDetailLambda := map[string][]namespaceDetail{} expectedNamespaceDetailLambda["AWS/Lambda"] = []namespaceDetail{ { @@ -269,7 +316,7 @@ func TestReadCloudwatchConfig(t *testing.T) { tags: []aws.Tag{ { Key: "name", - Value: "test-ec2", + Value: "test", }, }, }, @@ -282,7 +329,7 @@ func TestReadCloudwatchConfig(t *testing.T) { tags: []aws.Tag{ { Key: "name", - Value: "test-elb1", + Value: "test", }, }, }, @@ -293,7 +340,7 @@ func TestReadCloudwatchConfig(t *testing.T) { tags: []aws.Tag{ { Key: "name", - Value: "test-elb2", + Value: "test", }, }, }, @@ -303,6 +350,12 @@ func TestReadCloudwatchConfig(t *testing.T) { expectedNamespaceDetailELBLambda["AWS/Lambda"] = []namespaceDetail{ { statistics: defaultStatistics, + tags: []aws.Tag{ + { + Key: "name", + Value: "test", + }, + }, }, } expectedNamespaceDetailELBLambda["AWS/ELB"] = []namespaceDetail{ @@ -313,7 +366,7 @@ func TestReadCloudwatchConfig(t *testing.T) { tags: []aws.Tag{ { Key: "name", - Value: "test-elb1", + Value: "test", }, }, }, @@ -324,7 +377,7 @@ func TestReadCloudwatchConfig(t *testing.T) { tags: []aws.Tag{ { Key: "name", - Value: "test-elb2", + Value: "test", }, }, }, @@ -377,6 +430,7 @@ func TestReadCloudwatchConfig(t *testing.T) { cases := []struct { title string cloudwatchMetricsConfig []Config + tagsFilter []aws.Tag expectedListMetricDetailTotal listMetricWithDetail expectedNamespaceDetailTotal map[string][]namespaceDetail }{ @@ -396,6 +450,7 @@ func TestReadCloudwatchConfig(t *testing.T) { Statistic: []string{"Average"}, }, }, + nil, expectedListMetricWithDetailEC2, map[string][]namespaceDetail{}, }, @@ -418,6 +473,7 @@ func TestReadCloudwatchConfig(t *testing.T) { Namespace: "AWS/S3", }, }, + nil, expectedListMetricWithDetailEC2, expectedNamespaceWithDetailS3, }, @@ -456,6 +512,7 @@ func TestReadCloudwatchConfig(t *testing.T) { ResourceTypeFilter: "rds", }, }, + nil, expectedListMetricWithDetailEC2RDS, expectedNamespaceDetailLambda, }, @@ -472,6 +529,7 @@ func TestReadCloudwatchConfig(t *testing.T) { ResourceTypeFilter: "s3", }, }, + nil, listMetricWithDetail{ resourceTypeFilters: map[string][]aws.Tag{}, }, @@ -485,6 +543,7 @@ func TestReadCloudwatchConfig(t *testing.T) { ResourceTypeFilter: "ec2", }, }, + nil, listMetricWithDetail{ resourceTypeFilters: map[string][]aws.Tag{}, }, @@ -500,6 +559,7 @@ func TestReadCloudwatchConfig(t *testing.T) { Statistic: []string{"Average", "Maximum"}, }, }, + nil, listMetricWithDetail{ resourceTypeFilters: map[string][]aws.Tag{}, }, @@ -513,6 +573,7 @@ func TestReadCloudwatchConfig(t *testing.T) { MetricName: []string{"MemoryUsed"}, }, }, + nil, listMetricWithDetail{ resourceTypeFilters: map[string][]aws.Tag{}, }, @@ -525,36 +586,24 @@ func TestReadCloudwatchConfig(t *testing.T) { Namespace: "AWS/EC2", MetricName: []string{"CPUUtilization"}, ResourceTypeFilter: resourceTypeEC2, - Tags: []aws.Tag{ - { - Key: "name", - Value: "test-ec2", - }, - }, }, { Namespace: "AWS/ELB", MetricName: []string{"BackendConnectionErrors", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX"}, Statistic: []string{"Sum"}, ResourceTypeFilter: "elasticloadbalancing", - Tags: []aws.Tag{ - { - Key: "name", - Value: "test-elb1", - }, - }, }, { Namespace: "AWS/ELB", MetricName: []string{"HealthyHostCount", "SurgeQueueLength", "UnHealthyHostCount"}, Statistic: []string{"Maximum"}, ResourceTypeFilter: "elasticloadbalancing", - Tags: []aws.Tag{ - { - Key: "name", - Value: "test-elb2", - }, - }, + }, + }, + []aws.Tag{ + { + Key: "name", + Value: "test", }, }, listMetricWithDetail{ @@ -570,24 +619,12 @@ func TestReadCloudwatchConfig(t *testing.T) { MetricName: []string{"BackendConnectionErrors", "HTTPCode_Backend_2XX", "HTTPCode_Backend_3XX"}, Statistic: []string{"Sum"}, ResourceTypeFilter: "elasticloadbalancing", - Tags: []aws.Tag{ - { - Key: "name", - Value: "test-elb1", - }, - }, }, { Namespace: "AWS/ELB", MetricName: []string{"HealthyHostCount", "SurgeQueueLength", "UnHealthyHostCount"}, Statistic: []string{"Maximum"}, ResourceTypeFilter: "elasticloadbalancing", - Tags: []aws.Tag{ - { - Key: "name", - Value: "test-elb2", - }, - }, }, { Namespace: "AWS/Lambda", @@ -621,7 +658,13 @@ func TestReadCloudwatchConfig(t *testing.T) { ResourceTypeFilter: "rds", }, }, - expectedListMetricWithDetailEC2RDS, + []aws.Tag{ + { + Key: "name", + Value: "test", + }, + }, + expectedListMetricWithDetailEC2RDSWithTag, expectedNamespaceDetailELBLambda, }, { @@ -639,6 +682,7 @@ func TestReadCloudwatchConfig(t *testing.T) { ResourceTypeFilter: "ec2:instance", }, }, + nil, listMetricWithDetail{ resourceTypeFilters: map[string][]aws.Tag{}, }, @@ -660,6 +704,7 @@ func TestReadCloudwatchConfig(t *testing.T) { Statistic: []string{"Average"}, }, }, + nil, expectedListMetricsEC2WithDim, map[string][]namespaceDetail{}, }, @@ -668,6 +713,7 @@ func TestReadCloudwatchConfig(t *testing.T) { for _, c := range cases { t.Run(c.title, func(t *testing.T) { m.CloudwatchConfigs = c.cloudwatchMetricsConfig + m.MetricSet.TagsFilter = c.tagsFilter listMetricDetailTotal, namespaceDetailTotal := m.readCloudwatchConfig() assert.Equal(t, c.expectedListMetricDetailTotal, listMetricDetailTotal) assert.Equal(t, c.expectedNamespaceDetailTotal, namespaceDetailTotal)