Add azure storage metricset (#15342)

* create metricset * work in prog * Work on metricset * Add serviceType config option * Fix tests * Add func to retrieve on all dimensions * work on different intervals * revert custom event format * work on reducing api calls * fix fields and dashboards, work on reducing amount of api calls * Add json example * refactor * refactor * fix tests * fix tests * feedback
elastic · Jan 13, 2020 · ad8c503 · ad8c503
1 parent 9191b73
commit ad8c503
Show file tree

Hide file tree

Showing 36 changed files with 1,086 additions and 304 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -589,6 +589,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview dashboard for AWS SNS module {pull}14977[14977]
 - Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100]
 - Add a `system/service` metricset for systemd data. {pull}14206[14206]
+- Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
 - Add cost warnings for the azure module. {pull}15356[15356]
 - Release elb module as GA. {pull}15485[15485]
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -2131,84 +2131,84 @@ azure module
 
 
 
-*`azure.namespace`*::
-+
---
-The namespace selected
+[float]
+=== resource
 
+The resource specified
 
-type: keyword
 
---
 
-*`azure.subscription_id`*::
+*`azure.resource.name`*::
 +
 --
-The subscription ID
+The name of the resource
 
 
 type: keyword
 
 --
 
-*`azure.dimensions.*`*::
+*`azure.resource.type`*::
 +
 --
-Azure metric dimensions.
+The type of the resource
 
 
-type: object
+type: keyword
 
 --
 
-[float]
-=== resource
+*`azure.resource.group`*::
++
+--
+The resource group
 
-The resource specified
 
+type: keyword
 
+--
 
-*`azure.resource.name`*::
+*`azure.resource.tags.*`*::
 +
 --
-The name of the resource
+Azure resource tags.
 
 
-type: keyword
+type: object
 
 --
 
-*`azure.resource.type`*::
+*`azure.namespace`*::
 +
 --
-The type of the resource
+The namespace selected
 
 
 type: keyword
 
 --
 
-*`azure.resource.group`*::
+*`azure.subscription_id`*::
 +
 --
-The resource group
+The subscription ID
 
 
 type: keyword
 
 --
 
-*`azure.resource.tags.*`*::
+*`azure.dimensions.*`*::
 +
 --
-Azure resource tags.
+Azure metric dimensions.
 
 
 type: object
 
 --
 
-*`azure.resource.compute_vm.*.*`*::
+*`azure.compute_vm.*.*`*::
 +
 --
 compute_vm
@@ -2218,7 +2218,7 @@ type: object
 
 --
 
-*`azure.resource.compute_vm_scaleset.*.*`*::
+*`azure.compute_vm_scaleset.*.*`*::
 +
 --
 compute_vm_scaleset
@@ -2235,12 +2235,22 @@ monitor
 
 
 
-*`azure.resource.monitor.metrics.*.*`*::
+*`azure.monitor.metrics.*.*`*::
 +
 --
 Metrics returned.
 
 
+type: object
+
+--
+
+*`azure.storage.*.*`*::
++
+--
+storage account
+
+
 type: object
 
 --

diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc
@@ -78,6 +78,11 @@ so the `period` for `compute_vm` metricset  should be `300s` or multiples of `30
 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes,
 so the `period` for `compute_vm_scaleset` metricset  should be `300s` or multiples of `300s`.
 
+[float]
+=== `storage`
+This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes,
+so the `period` for `storage` metricset  should be `300s` or multiples of `300s`.
+
 
 [float]
 == Additional notes about metrics and costs
@@ -130,6 +135,16 @@ metricbeat.modules:
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 ----
 
 [float]
@@ -143,9 +158,13 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-azure-monitor,monitor>>
 
+* <<metricbeat-metricset-azure-storage,storage>>
+
 include::azure/compute_vm.asciidoc[]
 
 include::azure/compute_vm_scaleset.asciidoc[]
 
 include::azure/monitor.asciidoc[]
 
+include::azure/storage.asciidoc[]
+
diff --git a/metricbeat/docs/modules/azure/storage.asciidoc b/metricbeat/docs/modules/azure/storage.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-azure-storage]]
+=== azure storage metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/azure/storage/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-azure,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/azure/storage/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -28,9 +28,10 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-aws-sqs,sqs>>   
 |<<metricbeat-metricset-aws-usage,usage>> beta[]  
 |<<metricbeat-module-azure,azure>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.3+| .3+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
+.4+| .4+|  |<<metricbeat-metricset-azure-compute_vm,compute_vm>> beta[]  
 |<<metricbeat-metricset-azure-compute_vm_scaleset,compute_vm_scaleset>> beta[]  
 |<<metricbeat-metricset-azure-monitor,monitor>> beta[]  
+|<<metricbeat-metricset-azure-storage,storage>> beta[]  
 |<<metricbeat-module-beat,Beat>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .2+| .2+|  |<<metricbeat-metricset-beat-state,state>>   
 |<<metricbeat-metricset-beat-stats,stats>>   

diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -281,6 +281,16 @@ metricbeat.modules:
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
 #--------------------------------- Beat Module ---------------------------------
 - module: beat
   metricsets:

diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml
@@ -32,3 +32,13 @@
   client_secret: '${AZURE_CLIENT_SECRET:""}'
   tenant_id: '${AZURE_TENANT_ID:""}'
   subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+
+- module: azure
+  metricsets:
+  - storage
+  enabled: true
+  period: 300s
+  client_id: '${AZURE_CLIENT_ID:""}'
+  client_secret: '${AZURE_CLIENT_SECRET:""}'
+  tenant_id: '${AZURE_TENANT_ID:""}'
+  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml
@@ -35,3 +35,14 @@
 #  tenant_id: '${AZURE_TENANT_ID:""}'
 #  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
 #  refresh_list_interval: 600s
+
+#- module: azure
+#  metricsets:
+#  - storage
+#  enabled: true
+#  period: 300s
+#  client_id: '${AZURE_CLIENT_ID:""}'
+#  client_secret: '${AZURE_CLIENT_SECRET:""}'
+#  tenant_id: '${AZURE_TENANT_ID:""}'
+#  subscription_id: '${AZURE_SUBSCRIPTION_ID:""}'
+#  refresh_list_interval: 600s
diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc
@@ -68,6 +68,11 @@ so the `period` for `compute_vm` metricset  should be `300s` or multiples of `30
 This metricset will collect metrics from the virtual machine scalesets, these metrics will have a timegrain every 5 minutes,
 so the `period` for `compute_vm_scaleset` metricset  should be `300s` or multiples of `300s`.
 
+[float]
+=== `storage`
+This metricset will collect metrics from the storage accounts, these metrics will have a timegrain every 5 minutes,
+so the `period` for `storage` metricset  should be `300s` or multiples of `300s`.
+
 
 [float]
 == Additional notes about metrics and costs

diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml
@@ -4,44 +4,44 @@
     azure module
   release: beta
   fields:
-  - name: azure
-    type: group
-    description: >
-    fields:
-      - name: namespace
-        type: keyword
-        description: >
-          The namespace selected
-      - name: subscription_id
-        type: keyword
-        description: >
-          The subscription ID
-      - name: dimensions.*
-        type: object
-        object_type: keyword
-        object_type_mapping_type: "*"
-        description: >
-          Azure metric dimensions.
-      - name: resource
-        type: group
-        description: >
-          The resource specified
-        fields:
-        - name: name
-          type: keyword
+    - name: azure
+      type: group
+      description: >
+      fields:
+        - name: resource
+          type: group
           description: >
-            The name of the resource
-        - name: type
+            The resource specified
+          fields:
+            - name: name
+              type: keyword
+              description: >
+                The name of the resource
+            - name: type
+              type: keyword
+              description: >
+                The type of the resource
+            - name: group
+              type: keyword
+              description: >
+                The resource group
+            - name: tags.*
+              type: object
+              object_type: keyword
+              object_type_mapping_type: "*"
+              description: >
+                Azure resource tags.
+        - name: namespace
           type: keyword
           description: >
-            The type of the resource
-        - name: group
+            The namespace selected
+        - name: subscription_id
           type: keyword
           description: >
-            The resource group
-        - name: tags.*
+            The subscription ID
+        - name: dimensions.*
           type: object
           object_type: keyword
           object_type_mapping_type: "*"
           description: >
-            Azure resource tags.
+            Azure metric dimensions.