Collect cloudwatch stats using GetMetricData (influxdata#5544)

idohalevi · Sep 23, 2020 · 8b8ed28 · 8b8ed28
1 parent 3e0269c
commit 8b8ed28
Show file tree

Hide file tree

Showing 3 changed files with 556 additions and 273 deletions.
diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md
@@ -17,7 +17,7 @@ API endpoint. In the following order the plugin will attempt to authenticate.
 
 ```toml
 [[inputs.cloudwatch]]
-  ## Amazon Region (required)
+  ## Amazon Region
   region = "us-east-1"
 
   ## Amazon Credentials
@@ -28,12 +28,12 @@ API endpoint. In the following order the plugin will attempt to authenticate.
   ## 4) environment variables
   ## 5) shared credentials file
   ## 6) EC2 Instance Profile
-  #access_key = ""
-  #secret_key = ""
-  #token = ""
-  #role_arn = ""
-  #profile = ""
-  #shared_credential_file = ""
+  # access_key = ""
+  # secret_key = ""
+  # token = ""
+  # role_arn = ""
+  # profile = ""
+  # shared_credential_file = ""
 
   ## Endpoint to make request against, the correct endpoint is automatically
   ## determined and this option should only be set if you wish to override the
@@ -54,32 +54,43 @@ API endpoint. In the following order the plugin will attempt to authenticate.
   ## Collection Delay (required - must account for metrics availability via CloudWatch API)
   delay = "5m"
 
-  ## Override global run interval (optional - defaults to global interval)
-  ## Recomended: use metric 'interval' that is a multiple of 'period' to avoid
+  ## Recommended: use metric 'interval' that is a multiple of 'period' to avoid
   ## gaps or overlap in pulled data
   interval = "5m"
 
+  ## Configure the TTL for the internal cache of metrics.
+  # cache_ttl = "1h"
+
   ## Metric Statistic Namespace (required)
   namespace = "AWS/ELB"
 
   ## Maximum requests per second. Note that the global default AWS rate limit is
-  ## 400 reqs/sec, so if you define multiple namespaces, these should add up to a
-  ## maximum of 400. Optional - default value is 200.
+  ## 50 reqs/sec, so if you define multiple namespaces, these should add up to a
+  ## maximum of 50.
   ## See http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_limits.html
-  ratelimit = 200
+  # ratelimit = 25
+
+  ## Namespace-wide statistic filters. These allow fewer queries to be made to
+  ## cloudwatch.
+  # statistic_include = [ "average", "sum", minimum", "maximum", sample_count" ]
+  # statistic_exclude = []
 
-  ## Metrics to Pull (optional)
+  ## Metrics to Pull
   ## Defaults to all Metrics in Namespace if nothing is provided
   ## Refreshes Namespace available metrics every 1h
-  [[inputs.cloudwatch.metrics]]
-    names = ["Latency", "RequestCount"]
-
-    ## Dimension filters for Metric.  These are optional however all dimensions
-    ## defined for the metric names must be specified in order to retrieve
-    ## the metric statistics.
-    [[inputs.cloudwatch.metrics.dimensions]]
-      name = "LoadBalancerName"
-      value = "p-example"
+  #[[inputs.cloudwatch.metrics]]
+  #  names = ["Latency", "RequestCount"]
+  #
+  #  ## Statistic filters for Metric.  These allow for retrieving specific
+  #  ## statistics for an individual metric.
+  #  # statistic_include = [ "average", "sum", minimum", "maximum", sample_count" ]
+  #  # statistic_exclude = []
+  #
+  #  ## Dimension filters for Metric.  All dimensions defined for the metric names
+  #  ## must be specified in order to retrieve the metric statistics.
+  #  [[inputs.cloudwatch.metrics.dimensions]]
+  #    name = "LoadBalancerName"
+  #    value = "p-example"
 ```
 #### Requirements and Terminology
 
@@ -97,17 +108,21 @@ wildcard dimension is ignored.
 
 Example:
 ```
-[[inputs.cloudwatch.metrics]]
-  names = ["Latency"]
+[[inputs.cloudwatch]]
+  period = "1m"
+  interval = "5m"
 
-  ## Dimension filters for Metric (optional)
-  [[inputs.cloudwatch.metrics.dimensions]]
-    name = "LoadBalancerName"
-    value = "p-example"
+  [[inputs.cloudwatch.metrics]]
+    names = ["Latency"]
 
-  [[inputs.cloudwatch.metrics.dimensions]]
-    name = "AvailabilityZone"
-    value = "*"
+    ## Dimension filters for Metric (optional)
+    [[inputs.cloudwatch.metrics.dimensions]]
+      name = "LoadBalancerName"
+      value = "p-example"
+
+    [[inputs.cloudwatch.metrics.dimensions]]
+      name = "AvailabilityZone"
+      value = "*"
 ```
 
 If the following ELBs are available:
@@ -124,9 +139,11 @@ Then 2 metrics will be output:
 If the `AvailabilityZone` wildcard dimension was omitted, then a single metric (name: `p-example`)
 would be exported containing the aggregate values of the ELB across availability zones.
 
+To maximize efficiency and savings, consider making fewer requests by increasing `interval` but keeping `period` at the duration you would like metrics to be reported. The above example will request metrics from Cloudwatch every 5 minutes but will output five metrics timestamped one minute apart.
+
 #### Restrictions and Limitations
 - CloudWatch metrics are not available instantly via the CloudWatch API. You should adjust your collection `delay` to account for this lag in metrics availability based on your [monitoring subscription level](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html)
-- CloudWatch API usage incurs cost - see [GetMetricStatistics Pricing](https://aws.amazon.com/cloudwatch/pricing/)
+- CloudWatch API usage incurs cost - see [GetMetricData Pricing](https://aws.amazon.com/cloudwatch/pricing/)
 
 ### Measurements & Fields:
 
@@ -147,7 +164,6 @@ Tag Dimension names are represented in [snake case](https://en.wikipedia.org/wik
 
 - All measurements have the following tags:
   - region           (CloudWatch Region)
-  - unit             (CloudWatch Metric Unit)
   - {dimension-name} (Cloudwatch Dimension value - one for each metric dimension)
 
 ### Troubleshooting:
@@ -168,5 +184,5 @@ aws cloudwatch get-metric-statistics --namespace AWS/EC2 --region us-east-1 --pe
 
 ```
 $ ./telegraf --config telegraf.conf --input-filter cloudwatch --test
-> cloudwatch_aws_elb,load_balancer_name=p-example,region=us-east-1,unit=seconds latency_average=0.004810798017284538,latency_maximum=0.1100282669067383,latency_minimum=0.0006084442138671875,latency_sample_count=4029,latency_sum=19.382705211639404 1459542420000000000
+> cloudwatch_aws_elb,load_balancer_name=p-example,region=us-east-1 latency_average=0.004810798017284538,latency_maximum=0.1100282669067383,latency_minimum=0.0006084442138671875,latency_sample_count=4029,latency_sum=19.382705211639404 1459542420000000000
 ```