Cherry-pick #14971 to 7.x: Allow namespace: "*" for cloudwatch metric…

…set (#15119)

* Allow namespace: "*" for cloudwatch metricset (#14971)

* Allow namespace: "*" for cloudwatch metricset

(cherry picked from commit 83b309d)

* update changelog

CHANGELOG.next.asciidoc

@@ -524,6 +524,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Kafka JMX metricsets. {pull}14330[14330]
 - Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
 - Add module for ActiveMQ. {pull}14788[14788]
+- Enable wildcard for cloudwatch metricset namespace. {pull}14971[14971] {issue}14965[14965]
 - Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935]
 - Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934]
 - Add AWS SNS metricset. {pull}14946[14946]

x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc

@@ -16,7 +16,9 @@ iam:ListAccountAliases
 
 [float]
 === Metricset-specific configuration notes
-* *namespace*: The namespace to filter against. For example, AWS/EC2, AWS/S3.
+* *namespace*: The namespace used by ListMetrics API to filter against.
+For example, AWS/EC2, AWS/S3. If wildcard * is given for namespace, metrics
+from all namespaces will be collected automatically.
 * *name*: The name of the metric to filter against. For example, CPUUtilization for EC2 instance.
 * *dimensions*: The dimensions to filter against. For example, InstanceId=i-123.
 * *tags.resource_type_filter*: The constraints on the resources that you want returned.
@@ -28,12 +30,14 @@ only EC2 instances.
 By default, statistic includes Average, Sum, Count, Maximum and Minimum.
 
 [float]
-=== Configuration example
+=== Configuration examples
 To be more focused on `cloudwatch` metricset use cases, the examples below do
 not include configurations on AWS credentials.
 Please see <<aws-credentials-config,AWS credentials options>> for more details on setting AWS credentials
 in configurations in order for this metricset to make proper AWS API calls.
 
+[float]
+==== Example 1
 [source,yaml]
 ----
 - module: aws
@@ -66,6 +70,27 @@ specified and `dimensions` can be used to filter cloudwatch metrics. Please see
 https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/list-metrics.html[AWS List Metrics]
 for more details.
 
+[float]
+==== Example 2
+[source,yaml]
+----
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  metrics:
+    - namespace: "*"
+----
+With this config, metrics from all namespaces will be collected from Cloudwatch.
+The limitation here is the collection period for all namespaces are all set to
+be the same, which in this case is 300 second. This will cause extra costs for
+API calls or data loss.
+For example, metrics from namespace AWS/Usage are sent to Cloudwatch every 1
+minute. With the collection period equals to 300 seconds, data points in between
+will get lost. Metrics from namespace AWS/Billing are sent to Cloudwatch every
+several hours. By querying from AWS/Billing namespace every 300 seconds,
+additional costs will occur.
+
 [float]
 === More examples
 With the configuration below, users will be able to collect cloudwatch metrics

x-pack/metricbeat/module/aws/utils.go

@@ -41,7 +41,9 @@ func GetListMetricsOutput(namespace string, regionName string, svcCloudwatch clo
 		init = false
 		listMetricsInput := &cloudwatch.ListMetricsInput{
 			NextToken: nextToken,
-			Namespace: &namespace,
+		}
+		if namespace != "*" {
+			listMetricsInput.Namespace = &namespace
 		}
 		reqListMetrics := svcCloudwatch.ListMetricsRequest(listMetricsInput)
 

x-pack/metricbeat/module/aws/utils_test.go

@@ -163,6 +163,18 @@ func TestGetListMetricsOutput(t *testing.T) {
 	assert.Equal(t, instanceID, *listMetricsOutput[0].Dimensions[0].Value)
 }
 
+func TestGetListMetricsOutputWithWildcard(t *testing.T) {
+	svcCloudwatch := &MockCloudWatchClient{}
+	listMetricsOutput, err := GetListMetricsOutput("*", "us-west-1", svcCloudwatch)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(listMetricsOutput))
+	assert.Equal(t, namespace, *listMetricsOutput[0].Namespace)
+	assert.Equal(t, metricName, *listMetricsOutput[0].MetricName)
+	assert.Equal(t, 1, len(listMetricsOutput[0].Dimensions))
+	assert.Equal(t, dimName, *listMetricsOutput[0].Dimensions[0].Name)
+	assert.Equal(t, instanceID, *listMetricsOutput[0].Dimensions[0].Value)
+}
+
 func TestGetMetricDataPerRegion(t *testing.T) {
 	startTime, endTime := GetStartTimeEndTime(10 * time.Minute)