diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e2695013fa62..41b3721d06f7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -672,6 +672,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649] - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391] +- Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889] - Request prometheus endpoints to be gzipped by default {pull}20766[20766] - Release all kubernetes `state` metricsets as GA {pull}20901[20901] - Add billing metricset into googlecloud module. {pull}20812[20812] {issue}20738[20738] diff --git a/metricbeat/docs/modules/azure/compute_vm.asciidoc b/metricbeat/docs/modules/azure/compute_vm.asciidoc index fdac6f7d06a4..8715cf5c58e6 100644 --- a/metricbeat/docs/modules/azure/compute_vm.asciidoc +++ b/metricbeat/docs/modules/azure/compute_vm.asciidoc @@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go include::../../../../x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc[] +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. ==== Fields diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index b0b66085b2bc..ea6ab5697b0a 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -21,7 +21,6 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/billing" - _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/compute_vm_scaleset" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/storage" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-billing-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-billing-overview.json index b8ebdbdaf3c8..0776c3128e39 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-billing-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-billing-overview.json @@ -238,7 +238,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_4_index_pattern", "label": "Resource Name", @@ -483,7 +483,7 @@ "enabled": true, "id": "2", "params": { - "field": "cloud.instance.name", + "field": "azure.resource.name", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json index 711976480a1d..f29311dfbf8f 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-blob-storage-overview.json @@ -480,7 +480,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -510,7 +510,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -613,7 +613,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -765,7 +765,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1048,7 +1048,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-instance-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-instance-overview.json index 5a6fbac98636..92555a11e4d9 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-instance-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-instance-overview.json @@ -233,7 +233,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -306,7 +306,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -372,7 +372,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1584710535722", "indexPatternRefName": "control_2_index_pattern", "label": "Resource", @@ -471,7 +471,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -544,7 +544,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-registry-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-registry-overview.json index 030c9ec8b283..61516d560b77 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-registry-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-registry-overview.json @@ -233,7 +233,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -301,7 +301,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -367,7 +367,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1584710535722", "indexPatternRefName": "control_2_index_pattern", "label": "Resource", @@ -461,7 +461,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -529,7 +529,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-service-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-service-overview.json index 6d69131430b7..46ff5161f289 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-service-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-container-service-overview.json @@ -254,7 +254,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -322,7 +322,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries" } ], @@ -388,7 +388,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1584710535722", "indexPatternRefName": "control_2_index_pattern", "label": "Resource", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-database-account-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-database-account-overview.json index 32baee889f87..c0552fd8e8f8 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-database-account-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-database-account-overview.json @@ -438,7 +438,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1584710535722", "indexPatternRefName": "control_2_index_pattern", "label": "Resource", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json index 05e8f11b2806..b58598e38385 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-file-storage-overview.json @@ -520,7 +520,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -550,7 +550,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -653,7 +653,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -756,7 +756,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -908,7 +908,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1165,7 +1165,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1195,7 +1195,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -1272,7 +1272,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1302,7 +1302,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json index dc7b93f557d1..6226342295cd 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-queue-storage-overview.json @@ -480,7 +480,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -510,7 +510,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -613,7 +613,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -716,7 +716,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -868,7 +868,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json index 67157dd83ce5..0ad7aa93d44c 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-storage-overview.json @@ -474,7 +474,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -504,7 +504,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -607,7 +607,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -710,7 +710,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -861,7 +861,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", @@ -1118,7 +1118,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -1148,7 +1148,7 @@ "split_color_mode": "gradient", "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json index 8ee383609768..87d5bba84107 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-table-storage-overview.json @@ -480,7 +480,7 @@ "index_pattern": "metricbeat-*", "interval": "60m", "isModelInvalid": false, - "pivot_id": "cloud.instance.name", + "pivot_id": "azure.resource.name", "pivot_label": "Resource Name", "pivot_rows": "30", "pivot_type": "string", @@ -510,7 +510,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -613,7 +613,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -716,7 +716,7 @@ "split_color_mode": "gradient", "split_mode": "everything", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "terms_order_by": "61fb4192-32e4-11ea-b9f8-4d0b340ad993", "type": "timeseries", "value_template": "{{value}}" @@ -868,7 +868,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1578578146383", "indexPatternRefName": "control_2_index_pattern", "label": "Resource Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json index 0613a7e4a8ea..873616c609ca 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-guestmetrics-overview.json @@ -251,7 +251,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json index 08b5b0446332..e73bc6a1e5e1 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vm-overview.json @@ -266,7 +266,7 @@ "type": "list" }, { - "fieldName": "cloud.instance.name", + "fieldName": "azure.resource.name", "id": "1570774891724", "indexPatternRefName": "control_2_index_pattern", "label": "VM Name", @@ -372,7 +372,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "value_template": "{{value}}%" } ], @@ -447,7 +447,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "value_template": "{{value}}" } ], @@ -522,7 +522,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -598,7 +598,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "value_template": "{{value}}" } ], @@ -673,7 +673,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -749,7 +749,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -825,7 +825,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -901,7 +901,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json index 7e8ed312b79c..fe615da131fa 100644 --- a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-vmss-overview.json @@ -374,7 +374,7 @@ "split_color_mode": "rainbow", "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "value_template": "{{value}}" } ], @@ -449,7 +449,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "value_template": "{{value}}" } ], @@ -524,7 +524,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -600,7 +600,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -676,7 +676,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -752,7 +752,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -828,7 +828,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } @@ -904,7 +904,7 @@ "separate_axis": 0, "split_mode": "terms", "stacked": "none", - "terms_field": "cloud.instance.name", + "terms_field": "azure.resource.name", "type": "timeseries", "value_template": "{{value}}" } diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go new file mode 100644 index 000000000000..c8621e776922 --- /dev/null +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -0,0 +1,50 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func addHostMetadata(event *mb.Event, metricList common.MapStr) { + hostFieldTable := map[string]string{ + "percentage_cpu.avg": "host.cpu.pct", + "network_in_total.total": "host.network.in.bytes", + "network_in.total": "host.network.in.packets", + "network_out_total.total": "host.network.out.bytes", + "network_out.total": "host.network.out.packets", + "disk_read_bytes.total": "host.disk.read.bytes", + "disk_write_bytes.total": "host.disk.write.bytes", + } + + for metricName, hostName := range hostFieldTable { + metricValue, err := metricList.GetValue(metricName) + if err != nil { + continue + } + + if value, ok := metricValue.(float64); ok { + if metricName == "percentage_cpu.avg" { + value = value / 100 + } + event.RootFields.Put(hostName, value) + } + } +} + +func addCloudVMMetadata(event *mb.Event, resource Resource) { + event.RootFields.Put("cloud.instance.name", resource.Name) + event.RootFields.Put("host.name", resource.Name) + if resource.Vm != (VmResource{}) { + if resource.Vm.Id != "" { + event.RootFields.Put("cloud.instance.id", resource.Vm.Id) + event.RootFields.Put("host.id", resource.Vm.Id) + } + if resource.Vm.Size != "" { + event.RootFields.Put("cloud.machine.type", resource.Vm.Size) + } + } +} diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 44f098c96194..38db7f7c6d76 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -6,50 +6,12 @@ package azure import ( "fmt" - "time" "github.com/pkg/errors" "github.com/elastic/beats/v7/metricbeat/mb" ) -// Config options -type Config struct { - ClientId string `config:"client_id"` - ClientSecret string `config:"client_secret"` - TenantId string `config:"tenant_id"` - SubscriptionId string `config:"subscription_id"` - Period time.Duration `config:"period" validate:"nonzero,required"` - Resources []ResourceConfig `config:"resources"` - RefreshListInterval time.Duration `config:"refresh_list_interval"` - DefaultResourceType string `config:"default_resource_type"` -} - -// ResourceConfig contains resource and metric list specific configuration. -type ResourceConfig struct { - Id []string `config:"resource_id"` - Group []string `config:"resource_group"` - Metrics []MetricConfig `config:"metrics"` - Type string `config:"resource_type"` - Query string `config:"resource_query"` - ServiceType []string `config:"service_type"` -} - -// MetricConfig contains metric specific configuration. -type MetricConfig struct { - Name []string `config:"name"` - Namespace string `config:"namespace"` - Aggregations []string `config:"aggregations"` - Dimensions []DimensionConfig `config:"dimensions"` - Timegrain string `config:"timegrain"` -} - -// DimensionConfig contains dimensions specific configuration. -type DimensionConfig struct { - Name string `config:"name"` - Value string `config:"value"` -} - func init() { // Register the ModuleFactory function for the "azure" module. if err := mb.Registry.AddModule("azure", newModule); err != nil { @@ -127,20 +89,20 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // It publishes the event which is then forwarded to the output. In case // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { - err := m.Client.InitResources(m.MapMetrics, report) + err := m.Client.InitResources(m.MapMetrics) if err != nil { return err } - if len(m.Client.Resources.Metrics) == 0 { + if len(m.Client.ResourceConfigurations.Metrics) == 0 { // error message is previously logged in the InitResources, no error event should be created return nil } // retrieve metrics - groupedMetrics := groupMetricsByResource(m.Client.Resources.Metrics) + groupedMetrics := groupMetricsByResource(m.Client.ResourceConfigurations.Metrics) for _, metrics := range groupedMetrics { results := m.Client.GetMetricValues(metrics, report) - err := EventsMapping(results, m.BaseMetricSet.Name(), report) + err := EventsMapping(results, m.Client, report) if err != nil { return errors.Wrap(err, "error running EventsMapping") } @@ -160,19 +122,3 @@ func hasConfigOptions(config []string) bool { } return true } - -func (conf *Config) Validate() error { - if conf.SubscriptionId == "" { - return errors.New("no subscription ID has been configured") - } - if conf.ClientSecret == "" { - return errors.New("no client secret has been configured") - } - if conf.ClientId == "" { - return errors.New("no client ID has been configured") - } - if conf.TenantId == "" { - return errors.New("no tenant ID has been configured") - } - return nil -} diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index 57da9c3b23d0..5bfbbacca67c 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -20,10 +20,11 @@ import ( // Client represents the azure client which will make use of the azure sdk go metrics related clients type Client struct { - AzureMonitorService Service - Config Config - Resources ResourceConfiguration - Log *logp.Logger + AzureMonitorService Service + Config Config + ResourceConfigurations ResourceConfiguration + Log *logp.Logger + Resources []Resource } // mapResourceMetrics function type will map the configuration options to client metrics (depending on the metricset) @@ -40,21 +41,23 @@ func NewClient(config Config) (*Client, error) { Config: config, Log: logp.NewLogger("azure monitor client"), } - client.Resources.RefreshInterval = config.RefreshListInterval + client.ResourceConfigurations.RefreshInterval = config.RefreshListInterval return client, nil } // InitResources function will retrieve and validate the resources configured by the users and then map the information configured to client metrics. // the mapMetric function sent in this case will handle the mapping part as different metric and aggregation options work for different metricsets -func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) error { +func (client *Client) InitResources(fn mapResourceMetrics) error { if len(client.Config.Resources) == 0 { return errors.New("no resource options defined") } // check if refresh interval has been set and if it has expired - if !client.Resources.Expired() { + if !client.ResourceConfigurations.Expired() { return nil } var metrics []Metric + //reset client resources + client.Resources = []Resource{} for _, resource := range client.Config.Resources { // retrieve azure resources information resourceList, err := client.AzureMonitorService.GetResourceDefinitions(resource.Id, resource.Group, resource.Type, resource.Query) @@ -68,6 +71,19 @@ func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) client.Log.Error(err) continue } + //map resources to the client + for _, resource := range resourceList.Values() { + if !containsResource(*resource.ID, client.Resources) { + client.Resources = append(client.Resources, Resource{ + Id: *resource.ID, + Name: *resource.Name, + Location: *resource.Location, + Type: *resource.Type, + Group: getResourceGroupFromId(*resource.ID), + Tags: mapTags(resource.Tags), + Subscription: client.Config.SubscriptionId}) + } + } resourceMetrics, err := fn(client, resourceList.Values(), resource) if err != nil { return err @@ -79,7 +95,7 @@ func (client *Client) InitResources(fn mapResourceMetrics, report mb.ReporterV2) if len(metrics) == 0 { client.Log.Debug("no resources were found based on all the configurations options entered") } - client.Resources.Metrics = metrics + client.ResourceConfigurations.Metrics = metrics return nil } @@ -107,21 +123,21 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } filter = strings.Join(filterList, " AND ") } - resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.Resource.SubId, metric.Namespace, metric.TimeGrain, timespan, metric.Names, + resp, timegrain, err := client.AzureMonitorService.GetMetricValues(metric.ResourceSubId, metric.Namespace, metric.TimeGrain, timespan, metric.Names, metric.Aggregations, filter) if err != nil { - err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.Resource.SubId, metric.Namespace) + err = errors.Wrapf(err, "error while listing metric values by resource ID %s and namespace %s", metric.ResourceSubId, metric.Namespace) client.Log.Error(err) report.Error(err) } else { - for i, currentMetric := range client.Resources.Metrics { + for i, currentMetric := range client.ResourceConfigurations.Metrics { if matchMetrics(currentMetric, metric) { current := mapMetricValues(resp, currentMetric.Values, endTime.Truncate(time.Minute).Add(interval*(-1)), endTime.Truncate(time.Minute)) - client.Resources.Metrics[i].Values = current - if client.Resources.Metrics[i].TimeGrain == "" { - client.Resources.Metrics[i].TimeGrain = timegrain + client.ResourceConfigurations.Metrics[i].Values = current + if client.ResourceConfigurations.Metrics[i].TimeGrain == "" { + client.ResourceConfigurations.Metrics[i].TimeGrain = timegrain } - resultedMetrics = append(resultedMetrics, client.Resources.Metrics[i]) + resultedMetrics = append(resultedMetrics, client.ResourceConfigurations.Metrics[i]) } } } @@ -130,26 +146,20 @@ func (client *Client) GetMetricValues(metrics []Metric, report mb.ReporterV2) [] } // CreateMetric function will create a client metric based on the resource and metrics configured -func (client *Client) CreateMetric(selectedResourceID string, resource resources.GenericResource, resourceSize string, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { +func (client *Client) CreateMetric(resourceId string, subResourceId string, namespace string, metrics []string, aggregations string, dimensions []Dimension, timegrain string) Metric { + if subResourceId == "" { + subResourceId = resourceId + } met := Metric{ - Resource: Resource{ - SubId: selectedResourceID, - Id: *resource.ID, - Name: *resource.Name, - Location: *resource.Location, - Type: *resource.Type, - Group: getResourceGroupFromId(*resource.ID), - Tags: mapTags(resource.Tags), - Subscription: client.Config.SubscriptionId, - Size: resourceSize, - }, - Namespace: namespace, - Names: metrics, - Dimensions: dimensions, - Aggregations: aggregations, - TimeGrain: timegrain, + ResourceId: resourceId, + ResourceSubId: subResourceId, + Namespace: namespace, + Names: metrics, + Dimensions: dimensions, + Aggregations: aggregations, + TimeGrain: timegrain, } - for _, prevMet := range client.Resources.Metrics { + for _, prevMet := range client.ResourceConfigurations.Metrics { if len(prevMet.Values) != 0 && matchMetrics(prevMet, met) { met.Values = prevMet.Values } @@ -158,7 +168,7 @@ func (client *Client) CreateMetric(selectedResourceID string, resource resources } // MapMetricByPrimaryAggregation will map the primary aggregation of the metric definition to the client metric -func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDefinition, resource resources.GenericResource, selectedResourceID string, resourceSize string, namespace string, dim []Dimension, timegrain string) []Metric { +func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDefinition, resourceId string, subResourceId string, namespace string, dim []Dimension, timegrain string) []Metric { var clientMetrics []Metric metricGroups := make(map[string][]insights.MetricDefinition) @@ -170,10 +180,51 @@ func (client *Client) MapMetricByPrimaryAggregation(metrics []insights.MetricDef for _, metricName := range metricGroup { metricNames = append(metricNames, *metricName.Name.Value) } - if selectedResourceID == "" { - selectedResourceID = *resource.ID - } - clientMetrics = append(clientMetrics, client.CreateMetric(selectedResourceID, resource, resourceSize, namespace, metricNames, key, dim, timegrain)) + clientMetrics = append(clientMetrics, client.CreateMetric(resourceId, subResourceId, namespace, metricNames, key, dim, timegrain)) } return clientMetrics } + +// GetResourceForData will retrieve resource details for the selected metric configuration +func (client *Client) GetResourceForData(resourceId string) Resource { + for i, res := range client.Resources { + if res.Id == resourceId { + var vmSize string + var vmId string + if client.Config.AddCloudMetadata && res.Vm == (VmResource{}) { + expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(res.Id) + if err != nil { + client.Log.Error(err, "could not retrieve the resource details by resource ID %s", res.Id) + return Resource{} + } + if expandedResource.Properties != nil { + if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { + if hardware, ok := properties["hardwareProfile"]; ok { + if vmSz, ok := hardware.(map[string]interface{})["vmSize"]; ok { + vmSize = vmSz.(string) + } + if vmID, ok := properties["vmId"]; ok { + vmId = vmID.(string) + } + } + } + } + client.Resources[i].Vm = VmResource{Size: vmSize, Id: vmId} + return client.Resources[i] + } + return res + } + } + return Resource{} +} + +// NewMockClient instantiates a new client with the mock azure service +func NewMockClient() *Client { + azureMockService := new(MockService) + client := &Client{ + AzureMonitorService: azureMockService, + Config: Config{}, + Log: logp.NewLogger("test azure monitor"), + } + return client +} diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index 970a3d6e7946..47b88f99ccec 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -43,8 +43,7 @@ func mockMapResourceMetrics(client *Client, resources []resources.GenericResourc func TestInitResources(t *testing.T) { t.Run("return error when no resource options were configured", func(t *testing.T) { client := NewMockClient() - mr := MockReporterV2{} - err := client.InitResources(mockMapResourceMetrics, &mr) + err := client.InitResources(mockMapResourceMetrics) assert.Error(t, err, "no resource options were configured") }) t.Run("return error no resources were found", func(t *testing.T) { @@ -55,9 +54,9 @@ func TestInitResources(t *testing.T) { client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - err := client.InitResources(mockMapResourceMetrics, &mr) + err := client.InitResources(mockMapResourceMetrics) assert.Error(t, err, "no resources were found based on all the configurations options entered") - assert.Equal(t, len(client.Resources.Metrics), 0) + assert.Equal(t, len(client.ResourceConfigurations.Metrics), 0) m.AssertExpectations(t) }) } @@ -67,7 +66,7 @@ func TestGetMetricValues(t *testing.T) { client.Config = resourceIDConfig t.Run("return no error when no metric values are returned but log and send event", func(t *testing.T) { - client.Resources = ResourceConfiguration{ + client.ResourceConfigurations = ResourceConfiguration{ Metrics: []Metric{ { Namespace: "namespace", @@ -83,13 +82,13 @@ func TestGetMetricValues(t *testing.T) { client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - metrics := client.GetMetricValues(client.Resources.Metrics, &mr) + metrics := client.GetMetricValues(client.ResourceConfigurations.Metrics, &mr) assert.Equal(t, len(metrics), 0) - assert.Equal(t, len(client.Resources.Metrics[0].Values), 0) + assert.Equal(t, len(client.ResourceConfigurations.Metrics[0].Values), 0) m.AssertExpectations(t) }) t.Run("return metric values", func(t *testing.T) { - client.Resources = ResourceConfiguration{ + client.ResourceConfigurations = ResourceConfiguration{ Metrics: []Metric{ { Namespace: "namespace", @@ -105,9 +104,9 @@ func TestGetMetricValues(t *testing.T) { client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) - metricValues := client.GetMetricValues(client.Resources.Metrics, &mr) + metricValues := client.GetMetricValues(client.ResourceConfigurations.Metrics, &mr) assert.Equal(t, len(metricValues), 0) - assert.Equal(t, len(client.Resources.Metrics[0].Values), 0) + assert.Equal(t, len(client.ResourceConfigurations.Metrics[0].Values), 0) m.AssertExpectations(t) }) } diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index ebf2f7fc8128..c0cd02e589f2 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -5,7 +5,6 @@ package azure import ( - "fmt" "reflect" "strings" "time" @@ -87,7 +86,7 @@ func metricIsEmpty(metric insights.MetricValue) bool { // matchMetrics will compare current metrics func matchMetrics(prevMet Metric, met Metric) bool { - if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.Resource.Id == met.Resource.Id && + if prevMet.Namespace == met.Namespace && reflect.DeepEqual(prevMet.Names, met.Names) && prevMet.ResourceId == met.ResourceId && prevMet.Aggregations == met.Aggregations && prevMet.TimeGrain == met.TimeGrain { return true } @@ -105,27 +104,6 @@ func getResourceGroupFromId(path string) string { return "" } -// getResourceTypeFromId maps resource group from resource ID -func getResourceTypeFromId(path string) string { - params := strings.Split(path, "/") - for i, param := range params { - if param == "providers" { - return fmt.Sprintf("%s/%s", params[i+1], params[i+2]) - } - } - return "" -} - -// getResourceNameFormId maps resource group from resource ID -func getResourceNameFromId(path string) string { - params := strings.Split(path, "/") - if strings.HasSuffix(path, "/") { - return params[len(params)-2] - } - return params[len(params)-1] - -} - // mapTags maps resource tags func mapTags(azureTags map[string]*string) map[string]string { if len(azureTags) == 0 { @@ -181,10 +159,10 @@ func convertTimegrainToDuration(timegrain string) time.Duration { func groupMetricsByResource(metrics []Metric) map[string][]Metric { grouped := make(map[string][]Metric) for _, metric := range metrics { - if _, ok := grouped[metric.Resource.Id]; !ok { - grouped[metric.Resource.Id] = make([]Metric, 0) + if _, ok := grouped[metric.ResourceId]; !ok { + grouped[metric.ResourceId] = make([]Metric, 0) } - grouped[metric.Resource.Id] = append(grouped[metric.Resource.Id], metric) + grouped[metric.ResourceId] = append(grouped[metric.ResourceId], metric) } return grouped } @@ -198,3 +176,12 @@ func ContainsDimension(dimension string, dimensions []insights.LocalizableString } return false } + +func containsResource(resId string, resources []Resource) bool { + for _, res := range resources { + if res.Id == resId { + return true + } + } + return false +} diff --git a/x-pack/metricbeat/module/azure/client_utils_test.go b/x-pack/metricbeat/module/azure/client_utils_test.go index 323175c468e4..ffa09d4faeac 100644 --- a/x-pack/metricbeat/module/azure/client_utils_test.go +++ b/x-pack/metricbeat/module/azure/client_utils_test.go @@ -60,7 +60,7 @@ func TestMetricExists(t *testing.T) { func TestMatchMetrics(t *testing.T) { prev := Metric{ - Resource: Resource{Name: "vm", Group: "group", Id: "id"}, + ResourceId: "id", Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -69,7 +69,7 @@ func TestMatchMetrics(t *testing.T) { TimeGrain: "1PM", } current := Metric{ - Resource: Resource{Name: "vm", Group: "group", Id: "id"}, + ResourceId: "id", Namespace: "namespace", Names: []string{"TotalRequests,Capacity"}, Aggregations: "Average,Total", @@ -79,7 +79,7 @@ func TestMatchMetrics(t *testing.T) { } result := matchMetrics(prev, current) assert.True(t, result) - current.Resource.Id = "id1" + current.ResourceId = "id1" result = matchMetrics(prev, current) assert.False(t, result) } @@ -111,18 +111,6 @@ func TestGetResourceGroupFromID(t *testing.T) { assert.Equal(t, group, "obs-infrastructure") } -func TestGetResourceTypeFromID(t *testing.T) { - path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - rType := getResourceTypeFromId(path) - assert.Equal(t, rType, "Microsoft.Compute/virtualMachines") -} - -func TestGetResourceNameFromID(t *testing.T) { - path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" - name := getResourceNameFromId(path) - assert.Equal(t, name, "obstestmemleak") -} - func TestExpired(t *testing.T) { resConfig := ResourceConfiguration{} result := resConfig.Expired() diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 1665af3d455f..1da5cfb63abf 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,86 +2,62 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { - "cpu_credits_consumed": { - "avg": 0.01 - }, - "cpu_credits_remaining": { - "avg": 288 - }, - "disk_read_bytes": { - "total": 4019543.24 - }, "disk_read_operations_per_sec": { - "avg": 0.0605 - }, - "disk_write_bytes": { - "total": 16739967.27 + "avg": 3.3875 }, "disk_write_operations_per_sec": { - "avg": 1.9525 + "avg": 0.6705 }, "inbound_flows": { - "avg": 31 + "avg": 28.4 }, "inbound_flows_maximum_creation_rate": { - "avg": 0.8 - }, - "network_in": { - "total": 1501534 - }, - "network_in_total": { - "total": 1806936 - }, - "network_out": { - "total": 1647640 - }, - "network_out_total": { - "total": 3633130 + "avg": 10.4 }, "os_disk_queue_depth": { - "avg": 0 + "avg": 0.00125 }, "os_disk_read_bytes_per_sec": { - "avg": 13398.475 + "avg": 602589.1825 }, "os_disk_read_operations_per_sec": { - "avg": 0.121 + "avg": 5.28375 }, "os_disk_write_bytes_per_sec": { - "avg": 55799.886 + "avg": 14137.59375 }, "os_disk_write_operations_per_sec": { - "avg": 3.905 + "avg": 1.46875 }, "os_per_disk_qd": { - "avg": 0 + "avg": 0.00125 }, "os_per_disk_read_bytes_per_sec": { - "avg": 13398.475 + "avg": 602589.1825 }, "os_per_disk_read_operations_per_sec": { - "avg": 0.121 + "avg": 5.28375 }, "os_per_disk_write_bytes_per_sec": { - "avg": 55799.886 + "avg": 14137.59375 }, "os_per_disk_write_operations_per_sec": { - "avg": 3.905 + "avg": 1.46875 }, "outbound_flows": { - "avg": 31 + "avg": 28.4 }, "outbound_flows_maximum_creation_rate": { - "avg": 0.8 + "avg": 10.4 }, "per_disk_qd": { - "avg": 0 + "avg": 0.0025 }, "per_disk_read_bytes_per_sec": { - "avg": 0 + "avg": 51985.035 }, "per_disk_read_operations_per_sec": { - "avg": 0 + "avg": 2.92875 }, "per_disk_write_bytes_per_sec": { "avg": 0 @@ -90,30 +66,26 @@ "avg": 0 }, "percentage_cpu": { - "avg": 1.585 - }, - "premium_os_disk_cache_read_hit": { - "avg": 100 - }, - "premium_os_disk_cache_read_miss": { - "avg": 0 + "avg": 9.747 } }, "namespace": "Microsoft.Compute/virtualMachines", "resource": { - "group": "obs-test", + "group": "obs-infrastructure", + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux", + "name": "obslinux", "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", "timegrain": "PT5M" }, "cloud": { "instance": { - "id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test", - "name": "perfmon-test" + "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", + "name": "obslinux" }, "machine": { - "type": "Standard_B1ms" + "type": "Basic_A0" }, "provider": "azure", "region": "westeurope" @@ -125,28 +97,10 @@ }, "host": { "cpu": { - "pct": 0.01585 + "pct": 0.09747 }, - "disk": { - "read": { - "bytes": 4019543.24 - }, - "write": { - "bytes": 16739967.27 - } - }, - "id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test", - "name": "perfmon-test", - "network": { - "in": { - "bytes": 1806936, - "packets": 1501534 - }, - "out": { - "bytes": 3633130, - "packets": 1647640 - } - } + "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", + "name": "obslinux" }, "metricset": { "name": "compute_vm", @@ -155,4 +109,4 @@ "service": { "type": "azure" } -} \ No newline at end of file +} diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc index f30db5e06d85..2322de020c5c 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/docs.asciidoc @@ -8,7 +8,7 @@ include::../../_meta/shared-azure.asciidoc[] ==== Config options to identify resources `resource_id`:: (_[]string_) The fully qualified ID's of the resource, including the resource name and resource type. Has the format /subscriptions/{guid}/resourceGroups/{resource-group-name}/providers/{resource-provider-namespace}/{resource-type}/{resource-name}. - Should return a list of resources. +Should return a list of resources. `resource_group`:: (_[]string_) This option will select all virtual machines inside the resource group. diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper.go deleted file mode 100644 index dee4755d02bf..000000000000 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm - -import ( - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -// mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters -func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { - var metrics []azure.Metric - if len(resourceConfig.Metrics) == 0 { - return nil, nil - } - for _, resource := range resources { - // return resource size - resourceSize := mapResourceSize(resource, client) - // return all namespaces supported for this resource - namespaces, err := client.AzureMonitorService.GetMetricNamespaces(*resource.ID) - if err != nil { - return nil, errors.Wrapf(err, "no metric namespaces were found for resource %s", *resource.ID) - } - for _, namespace := range *namespaces.Value { - // get all metric definitions supported by the namespace provided - metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, *namespace.Properties.MetricNamespaceName) - if err != nil { - return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) - } - if len(*metricDefinitions.Value) == 0 { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, *namespace.Properties.MetricNamespaceName) - } - var filteredMetricDefinitions []insights.MetricDefinition - for _, metricDefinition := range *metricDefinitions.Value { - filteredMetricDefinitions = append(filteredMetricDefinitions, metricDefinition) - } - // map azure metric definitions to client metrics - metrics = append(metrics, client.MapMetricByPrimaryAggregation(filteredMetricDefinitions, resource, "", resourceSize, *namespace.Properties.MetricNamespaceName, nil, azure.DefaultTimeGrain)...) - } - } - return metrics, nil -} - -// mapResourceSize func will try to map if existing the resource size -func mapResourceSize(resource resources.GenericResource, client *azure.Client) string { - if resource.Sku != nil && resource.Sku.Name != nil { - return *resource.Sku.Name - } - if resource.Sku == nil && resource.Properties == nil { - expandedResource, err := client.AzureMonitorService.GetResourceDefinitionById(*resource.ID) - if err != nil { - client.Log.Error(err, "could not retrieve the resource details by resource ID %s", *resource.ID) - return "" - } - if expandedResource.Properties != nil { - if properties, ok := expandedResource.Properties.(map[string]interface{}); ok { - if hardware, ok := properties["hardwareProfile"]; ok { - if vmSize, ok := hardware.(map[string]interface{})["vmSize"]; ok { - return vmSize.(string) - } - } - } - } - } - return "" -} diff --git a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go deleted file mode 100644 index 8a145dc95b25..000000000000 --- a/x-pack/metricbeat/module/azure/compute_vm/client_helper_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm - -import ( - "testing" - - "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -func MockResource() resources.GenericResource { - id := "123" - name := "resourceName" - location := "resourceLocation" - rType := "resourceType" - return resources.GenericResource{ - ID: &id, - Name: &name, - Location: &location, - Type: &rType, - } -} - -func MockNamespace() insights.MetricNamespaceCollection { - name := "namespace" - property := insights.MetricNamespaceName{ - MetricNamespaceName: &name, - } - namespace := insights.MetricNamespace{ - Name: &name, - Properties: &property, - } - list := []insights.MetricNamespace{namespace} - return insights.MetricNamespaceCollection{ - Value: &list, - } -} - -func MockMetricDefinitions() *[]insights.MetricDefinition { - metric1 := "TotalRequests" - metric2 := "Capacity" - metric3 := "BytesRead" - defs := []insights.MetricDefinition{ - { - Name: &insights.LocalizableString{Value: &metric1}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Maximum, insights.Count, insights.Total, insights.Average}, - }, - { - Name: &insights.LocalizableString{Value: &metric2}, - PrimaryAggregationType: insights.Average, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - }, - { - Name: &insights.LocalizableString{Value: &metric3}, - PrimaryAggregationType: insights.Minimum, - SupportedAggregationTypes: &[]insights.AggregationType{insights.Average, insights.Count, insights.Minimum}, - }, - } - return &defs -} - -func TestMapMetric(t *testing.T) { - resource := MockResource() - namespace := MockNamespace() - metricDefinitions := insights.MetricDefinitionCollection{ - Value: MockMetricDefinitions(), - } - emptyList := []insights.MetricDefinition{} - emptyMetricDefinitions := insights.MetricDefinitionCollection{ - Value: &emptyList, - } - metricConfig := azure.MetricConfig{Name: []string{"*"}} - var resourceConfig = azure.ResourceConfig{Metrics: []azure.MetricConfig{metricConfig}} - client := azure.NewMockClient() - res := resource - res.Properties = map[string]interface{}{"hardwareProfile": map[string]interface{}{"vmSize": "A1Basic"}} - t.Run("return error when the metric namespaces api call returns an error", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) - m.On("GetMetricNamespaces", mock.Anything).Return(insights.MetricNamespaceCollection{}, errors.New("invalid resource ID")) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.Error(t, err) - assert.Equal(t, err.Error(), "no metric namespaces were found for resource 123: invalid resource ID") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return error when no metric definitions were found", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) - m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(emptyMetricDefinitions, nil) - client.AzureMonitorService = m - metric, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.Error(t, err) - assert.Equal(t, err.Error(), "no metric definitions were found for resource 123 and namespace namespace.") - assert.Equal(t, metric, []azure.Metric(nil)) - m.AssertExpectations(t) - }) - t.Run("return mapped metrics correctly", func(t *testing.T) { - m := &azure.MockService{} - m.On("GetResourceDefinitionById", mock.Anything).Return(res, nil) - m.On("GetMetricNamespaces", mock.Anything).Return(namespace, nil) - m.On("GetMetricDefinitions", mock.Anything, mock.Anything).Return(metricDefinitions, nil) - client.AzureMonitorService = m - metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) - assert.NoError(t, err) - assert.Equal(t, metrics[0].Resource.Id, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") - assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.Id, "123") - assert.Equal(t, metrics[1].Resource.Name, "resourceName") - assert.Equal(t, metrics[1].Resource.Type, "resourceType") - assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") - assert.Equal(t, metrics[1].Resource.Size, "A1Basic") - assert.Equal(t, metrics[1].Namespace, "namespace") - assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) - assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) - - //order of elements can be different when running the test - if metrics[0].Aggregations == "Average" { - assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity"}) - } else { - assert.Equal(t, metrics[0].Names, []string{"BytesRead"}) - assert.Equal(t, metrics[0].Aggregations, "Minimum") - } - - m.AssertExpectations(t) - }) -} diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go deleted file mode 100644 index a432ed610e76..000000000000 --- a/x-pack/metricbeat/module/azure/compute_vm/compute_vm.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package compute_vm - -import ( - "fmt" - - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" -) - -const defaultVMNamespace = "Microsoft.Compute/virtualMachines" - -// init registers the MetricSet with the central registry as soon as the program -// starts. The New function will be called later to instantiate an instance of -// the MetricSet for each host defined in the module's configuration. After the -// MetricSet has been created then Fetch will begin to be called periodically. -func init() { - mb.Registry.MustAddMetricSet("azure", "compute_vm", New) -} - -// MetricSet holds any configuration or state information. It must implement -// the mb.MetricSet interface. And this is best achieved by embedding -// mb.BaseMetricSet because it implements all of the required mb.MetricSet -// interface methods except for Fetch. -type MetricSet struct { - *azure.MetricSet -} - -// New creates a new instance of the MetricSet. New is responsible for unpacking -// any MetricSet specific configuration options if there are any. -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - ms, err := azure.NewMetricSet(base) - if err != nil { - return nil, err - } - // if no options are entered we will retrieve all the vm's from the entire subscription - if len(ms.Client.Config.Resources) == 0 { - ms.Client.Config.Resources = []azure.ResourceConfig{ - { - Query: fmt.Sprintf("resourceType eq '%s'", defaultVMNamespace), - }, - } - } - for index := range ms.Client.Config.Resources { - // if any resource groups were configured the resource type should be added - if len(ms.Client.Config.Resources[index].Group) > 0 { - ms.Client.Config.Resources[index].Type = defaultVMNamespace - } - // one metric configuration will be added containing all metrics names - ms.Client.Config.Resources[index].Metrics = []azure.MetricConfig{ - { - Name: []string{"*"}, - }, - } - } - ms.MapMetrics = mapMetrics - return &MetricSet{ - MetricSet: ms, - }, nil -} diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go index 2da62daaeec6..929e4e98a828 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_integration_test.go @@ -15,6 +15,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_test.go b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_test.go index 80d5033cbc11..5c456aab8618 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/compute_vm_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm/compute_vm_test.go @@ -5,70 +5,13 @@ package compute_vm import ( - "fmt" - "testing" + "os" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) -var ( - missingResourcesConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"compute_vm"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - } - - resourceConfig = common.MapStr{ - "module": "azure", - "period": "60s", - "metricsets": []string{"compute_vm"}, - "client_secret": "unique identifier", - "client_id": "unique identifier", - "subscription_id": "unique identifier", - "tenant_id": "unique identifier", - "resources": []common.MapStr{ - { - "resource_id": "test", - "metrics": []map[string]interface{}{ - { - "name": []string{"*"}, - }}, - }}, - } -) - -func TestFetch(t *testing.T) { - c, err := common.NewConfigFrom(missingResourcesConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err := mb.NewModule(c, mb.Registry) - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - assert.NoError(t, err) - ms, ok := metricsets[0].(*MetricSet) - assert.Equal(t, len(ms.Client.Config.Resources), 1) - assert.Equal(t, ms.Client.Config.Resources[0].Query, fmt.Sprintf("resourceType eq '%s'", defaultVMNamespace)) - - c, err = common.NewConfigFrom(resourceConfig) - if err != nil { - t.Fatal(err) - } - module, metricsets, err = mb.NewModule(c, mb.Registry) - if err != nil { - t.Fatal(err) - } - assert.NotNil(t, module) - assert.NotNil(t, metricsets) - ms, ok = metricsets[0].(*MetricSet) - require.True(t, ok, "metricset must be MetricSet") - assert.NotNil(t, ms) +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) } diff --git a/x-pack/metricbeat/module/azure/compute_vm/manifest.yml b/x-pack/metricbeat/module/azure/compute_vm/manifest.yml new file mode 100644 index 000000000000..4cfcad00cc75 --- /dev/null +++ b/x-pack/metricbeat/module/azure/compute_vm/manifest.yml @@ -0,0 +1,28 @@ +default: true +input: + module: azure + metricset: monitor + defaults: + default_resource_type: "Microsoft.Compute/virtualMachines" + add_cloud_metadata: true + resources: + - resource_group: "" + resource_type: "Microsoft.Compute/virtualMachines" + metrics: + - name: "*" + namespace: "Microsoft.Compute/virtualMachines" + timegrain: "PT5M" + - name: "*" + namespace: "Azure.VM.Windows.GuestMetrics" + timegrain: "PT5M" + ignore_unsupported: true + - resource_id: "" + timegrain: "PT5M" + metrics: + - name: "*" + namespace: "Microsoft.Compute/virtualMachines" + timegrain: "PT5M" + - name: "*" + namespace: "Azure.VM.Windows.GuestMetrics" + timegrain: "PT5M" + ignore_unsupported: true diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index 8edb9c5b69ac..6baa47d8e8c5 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -1,45 +1,82 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { - "compute_vm_scaleset": { - "cpu_credits_consumed": { - "avg": 0.019 + "dimensions": { + "vmname": "obslinuxvmss_0" + }, + "metrics": { + "disk_read_bytes": { + "total": 0 + }, + "disk_read_operations_per_sec": { + "avg": 0 + }, + "disk_write_bytes": { + "total": 1032189.54 + }, + "disk_write_operations_per_sec": { + "avg": 0.254 + }, + "inbound_flows": { + "avg": 58.2 + }, + "inbound_flows_maximum_creation_rate": { + "avg": 8.4 + }, + "network_in": { + "total": 253142 + }, + "network_in_total": { + "total": 646125 + }, + "network_out": { + "total": 874827 + }, + "network_out_total": { + "total": 1140051 + }, + "os_disk_bandwidth_consumed_percentage": { + "avg": 0 }, - "cpu_credits_remaining": { - "avg": 97.99 + "os_disk_iops_consumed_percentage": { + "avg": 0 }, - "os_per_disk_qd": { + "os_disk_queue_depth": { "avg": 0 }, - "os_per_disk_read_bytes_per_sec": { + "os_disk_read_bytes_per_sec": { "avg": 0 }, - "os_per_disk_read_operations_per_sec": { + "os_disk_read_operations_per_sec": { "avg": 0 }, - "os_per_disk_write_bytes_per_sec": { - "avg": 19201.653 + "os_disk_write_bytes_per_sec": { + "avg": 3440.626 + }, + "os_disk_write_operations_per_sec": { + "avg": 0.508 + }, + "outbound_flows": { + "avg": 58.2 + }, + "outbound_flows_maximum_creation_rate": { + "avg": 11.2 }, - "os_per_disk_write_operations_per_sec": { - "avg": 1.0366666666666666 + "percentage_cpu": { + "avg": 0.966 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { "group": "obs-infrastructure", + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss", "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", "timegrain": "PT5M" }, "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", - "name": "obslinuxvmss" - }, - "machine": { - "type": "Standard_B1ls" - }, "provider": "azure", "region": "westeurope" }, @@ -48,6 +85,29 @@ "duration": 115000, "module": "azure" }, + "host": { + "cpu": { + "pct": 0.00966 + }, + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 1032189.54 + } + }, + "network": { + "in": { + "bytes": 646125, + "packets": 253142 + }, + "out": { + "bytes": 1140051, + "packets": 874827 + } + } + }, "metricset": { "name": "compute_vm_scaleset", "period": 10000 diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go index f45896dce8eb..ecabff747419 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper.go @@ -25,7 +25,6 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res var metrics []azure.Metric for _, resource := range resources { // return resource size - resourceSize := mapResourceSize(resource) for _, metric := range resourceConfig.Metrics { metricDefinitions, err := client.AzureMonitorService.GetMetricDefinitions(*resource.ID, metric.Namespace) if err != nil { @@ -77,7 +76,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if key != azure.NoDimension { dimensions = []azure.Dimension{{Name: key, Value: "*"}} } - metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, resource, "", resourceSize, metric.Namespace, dimensions, azure.DefaultTimeGrain)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(metricGroup, *resource.ID, "", metric.Namespace, dimensions, azure.DefaultTimeGrain)...) } } } diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go index c1b0525294da..764cef2962b8 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/client_helper_test.go @@ -98,16 +98,9 @@ func TestMapMetric(t *testing.T) { assert.NoError(t, err) assert.Equal(t, len(metrics), 2) - assert.Equal(t, metrics[0].Resource.Id, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].ResourceId, "123") assert.Equal(t, metrics[0].Namespace, "namespace") - assert.Equal(t, metrics[1].Resource.Id, "123") - assert.Equal(t, metrics[1].Resource.Name, "resourceName") - assert.Equal(t, metrics[1].Resource.Type, "resourceType") - assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") - assert.Equal(t, metrics[1].Resource.Size, "standard") + assert.Equal(t, metrics[1].ResourceId, "123") assert.Equal(t, metrics[1].Namespace, "namespace") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil)) diff --git a/x-pack/metricbeat/module/azure/config.go b/x-pack/metricbeat/module/azure/config.go new file mode 100644 index 000000000000..63bb5450b571 --- /dev/null +++ b/x-pack/metricbeat/module/azure/config.go @@ -0,0 +1,68 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "time" + + "github.com/pkg/errors" +) + +// Config options +type Config struct { + ClientId string `config:"client_id"` + ClientSecret string `config:"client_secret"` + TenantId string `config:"tenant_id"` + SubscriptionId string `config:"subscription_id"` + Period time.Duration `config:"period" validate:"nonzero,required"` + Resources []ResourceConfig `config:"resources"` + RefreshListInterval time.Duration `config:"refresh_list_interval"` + DefaultResourceType string `config:"default_resource_type"` + AddCloudMetadata bool `config:"add_cloud_metadata"` +} + +// ResourceConfig contains resource and metric list specific configuration. +type ResourceConfig struct { + Id []string `config:"resource_id"` + Group []string `config:"resource_group"` + Metrics []MetricConfig `config:"metrics"` + Type string `config:"resource_type"` + Query string `config:"resource_query"` + ServiceType []string `config:"service_type"` +} + +// MetricConfig contains metric specific configuration. +type MetricConfig struct { + Name []string `config:"name"` + Namespace string `config:"namespace"` + Aggregations []string `config:"aggregations"` + Dimensions []DimensionConfig `config:"dimensions"` + Timegrain string `config:"timegrain"` + // namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if namespace is unsupported + // info messages will be logged instead + IgnoreUnsupported bool `config:"ignore_unsupported"` +} + +// DimensionConfig contains dimensions specific configuration. +type DimensionConfig struct { + Name string `config:"name"` + Value string `config:"value"` +} + +func (conf *Config) Validate() error { + if conf.SubscriptionId == "" { + return errors.New("no subscription ID has been configured") + } + if conf.ClientSecret == "" { + return errors.New("no client secret has been configured") + } + if conf.ClientId == "" { + return errors.New("no client ID has been configured") + } + if conf.TenantId == "" { + return errors.New("no tenant ID has been configured") + } + return nil +} diff --git a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json index aec7b3bdffba..945e31e3ac91 100644 --- a/x-pack/metricbeat/module/azure/container_instance/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_instance/_meta/data.json @@ -6,28 +6,23 @@ "avg": 0 }, "memory_usage": { - "avg": 0 + "avg": 2666496 } }, "dimensions": { - "container_name": "testcontainergroup" + "container_name": "anothercontainer" }, "namespace": "Microsoft.ContainerInstance/containerGroups", "resource": { "group": "obs-infrastructure", - "tags": { - "tag1": "value1" - }, + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerInstance/containerGroups/anothercontainer", + "name": "anothercontainer", "type": "Microsoft.ContainerInstance/containerGroups" }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", "timegrain": "PT5M" }, "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerInstance/containerGroups/testcontainergroup", - "name": "testcontainergroup" - }, "provider": "azure", "region": "westeurope" }, diff --git a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json index 0ae5b5f3e259..6b57c49b4eb9 100644 --- a/x-pack/metricbeat/module/azure/container_registry/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_registry/_meta/data.json @@ -18,16 +18,14 @@ "namespace": "Microsoft.ContainerRegistry/registries", "resource": { "group": "obs-infrastructure", - "type": "Microsoft.ContainerRegistry/registries" + "type": "Microsoft.ContainerRegistry/registries", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerRegistry/registries/obstest", + "name": "obstest" }, "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerRegistry/registries/obstest", - "name": "obstest" - }, "provider": "azure", "region": "westeurope" }, diff --git a/x-pack/metricbeat/module/azure/container_service/_meta/data.json b/x-pack/metricbeat/module/azure/container_service/_meta/data.json index 07d8025c93da..66cdca124dfd 100644 --- a/x-pack/metricbeat/module/azure/container_service/_meta/data.json +++ b/x-pack/metricbeat/module/azure/container_service/_meta/data.json @@ -7,21 +7,21 @@ } }, "dimensions": { - "status": "true" + "condition": "PIDPressure", + "node": "aks-agentpool-38582116-vmss000000", + "status": "false" }, "namespace": "Microsoft.ContainerService/managedClusters", "resource": { "group": "obs-infrastructure", + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerService/managedClusters/obskube", + "name": "obskube", "type": "Microsoft.ContainerService/managedClusters" }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", "timegrain": "PT5M" }, "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.ContainerService/managedClusters/obskube", - "name": "obskube" - }, "provider": "azure", "region": "westeurope" }, diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 936a4887f713..d0ba18fdc365 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -23,7 +23,7 @@ const ( ) // EventsMapping will map metric values to beats events -func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) error { +func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error { // metrics and metric values are currently grouped relevant to the azure REST API calls (metrics with the same aggregations per call) // multiple metrics can be mapped in one event depending on the resource, namespace, dimensions and timestamp @@ -35,7 +35,7 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err continue } // build a resource key with unique resource namespace combination - resNamkey := fmt.Sprintf("%s,%s", metric.Resource.Id, metric.Namespace) + resNamkey := fmt.Sprintf("%s,%s", metric.ResourceId, metric.Namespace) groupByResourceNamespace[resNamkey] = append(groupByResourceNamespace[resNamkey], metric) } // grouping metrics by the dimensions configured @@ -58,6 +58,7 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err // grouping metric values by timestamp and creating events (for each metric the REST api can retrieve multiple metric values for same aggregation but different timeframes) for _, grouped := range groupByDimensions { defaultMetric := grouped[0] + resource := client.GetResourceForData(defaultMetric.ResourceId) groupByTimeMetrics := make(map[time.Time][]MetricValue) for _, metric := range grouped { for _, m := range metric.Values { @@ -77,13 +78,16 @@ func EventsMapping(metrics []Metric, metricset string, report mb.ReporterV2) err groupByDimensions[dimKey] = append(groupByDimensions[dimKey], dimGroupValue) } for _, groupDimValues := range groupByDimensions { - event, metricList = createEvent(timestamp, defaultMetric, groupDimValues) + event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) } } } else { - event, metricList = createEvent(timestamp, defaultMetric, groupTimeValues) + event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) } - if metricset == nativeMetricset { + if client.Config.AddCloudMetadata { + addCloudVMMetadata(&event, resource) + } + if client.Config.DefaultResourceType == "" { event.ModuleFields.Put("metrics", metricList) } else { for key, metric := range metricList { @@ -142,23 +146,37 @@ func replaceUpperCase(src string) string { } // createEvent will create a new base event -func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) (mb.Event, common.MapStr) { +func createEvent(timestamp time.Time, metric Metric, resource Resource, metricValues []MetricValue) (mb.Event, common.MapStr) { + event := mb.Event{ ModuleFields: common.MapStr{ "timegrain": metric.TimeGrain, + "namespace": metric.Namespace, "resource": common.MapStr{ - "type": metric.Resource.Type, - "group": metric.Resource.Group, + "type": resource.Type, + "group": resource.Group, + "name": resource.Name, }, - "subscription_id": metric.Resource.Subscription, - "namespace": metric.Namespace, + "subscription_id": resource.Subscription, }, MetricSetFields: common.MapStr{}, Timestamp: timestamp, + RootFields: common.MapStr{ + "cloud": common.MapStr{ + "provider": "azure", + "region": resource.Location, + }, + }, + } + if metric.ResourceSubId != "" { + event.ModuleFields.Put("resource.id", metric.ResourceSubId) + } else { + event.ModuleFields.Put("resource.id", resource.Id) } - if len(metric.Resource.Tags) > 0 { - event.ModuleFields.Put("resource.tags", metric.Resource.Tags) + if len(resource.Tags) > 0 { + event.ModuleFields.Put("resource.tags", resource.Tags) } + if len(metric.Dimensions) > 0 { for _, dimension := range metric.Dimensions { if dimension.Value == "*" { @@ -169,21 +187,7 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) } } - event.RootFields = common.MapStr{} - event.RootFields.Put("cloud.provider", "azure") - event.RootFields.Put("cloud.region", metric.Resource.Location) - event.RootFields.Put("cloud.instance.name", metric.Resource.Name) - event.RootFields.Put("host.name", metric.Resource.Name) - if metric.Resource.SubId != "" { - event.RootFields.Put("cloud.instance.id", metric.Resource.SubId) - event.RootFields.Put("host.id", metric.Resource.SubId) - } else { - event.RootFields.Put("cloud.instance.id", metric.Resource.Id) - event.RootFields.Put("host.id", metric.Resource.Id) - } - if metric.Resource.Size != "" { - event.RootFields.Put("cloud.machine.type", metric.Resource.Size) - } + metricList := common.MapStr{} for _, value := range metricValues { metricNameString := fmt.Sprintf("%s", managePropertyName(value.name)) @@ -203,7 +207,7 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "count"), *value.count) } } - event = addHostFields(event, metricList) + addHostMetadata(&event, metricList) return event, metricList } @@ -233,30 +237,3 @@ func returnAllDimensions(dimensions []Dimension) (bool, []Dimension) { } return true, dims } - -func addHostFields(event mb.Event, metricList common.MapStr) mb.Event { - hostFieldTable := map[string]string{ - "percentage_cpu.avg": "host.cpu.pct", - "network_in_total.total": "host.network.in.bytes", - "network_in.total": "host.network.in.packets", - "network_out_total.total": "host.network.out.bytes", - "network_out.total": "host.network.out.packets", - "disk_read_bytes.total": "host.disk.read.bytes", - "disk_write_bytes.total": "host.disk.write.bytes", - } - - for metricName, hostName := range hostFieldTable { - metricValue, err := metricList.GetValue(metricName) - if err != nil { - continue - } - - if value, ok := metricValue.(float64); ok { - if metricName == "percentage_cpu.avg" { - value = value / 100 - } - event.RootFields.Put(hostName, value) - } - } - return event -} diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index 1ef042b12c2e..cdfad1965f8d 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -73,16 +73,17 @@ func TestCreateEvent(t *testing.T) { if !assert.NoError(t, err) { t.Fatal(err) } + resource := Resource{ + Id: "resId", + Name: "res", + Location: "west_europe", + Type: "resType", + Group: "resGroup", + Tags: nil, + Subscription: "subId", + } metric := Metric{ - Resource: Resource{ - Id: "resId", - Name: "res", - Location: "west_europe", - Type: "resType", - Group: "resGroup", - Tags: nil, - Subscription: "subId", - }, + ResourceId: "resId", Namespace: "namespace1", Names: []string{"Percentage CPU"}, Aggregations: "", @@ -103,7 +104,7 @@ func TestCreateEvent(t *testing.T) { dimensions: nil, }, } - event, list := createEvent(createTime, metric, metricValues) + event, list := createEvent(createTime, metric, resource, metricValues) assert.NotNil(t, event) assert.NotNil(t, list) assert.Equal(t, event.Timestamp, createTime) @@ -111,7 +112,7 @@ func TestCreateEvent(t *testing.T) { if !assert.NoError(t, err) { t.Fatal(err) } - assert.Equal(t, sub, metric.Resource.Subscription) + assert.Equal(t, sub, resource.Subscription) namespace, err := event.ModuleFields.GetValue("namespace") if !assert.NoError(t, err) { t.Fatal(err) diff --git a/x-pack/metricbeat/module/azure/database_account/_meta/data.json b/x-pack/metricbeat/module/azure/database_account/_meta/data.json index 68dbb92043cf..35ee448fd4d8 100644 --- a/x-pack/metricbeat/module/azure/database_account/_meta/data.json +++ b/x-pack/metricbeat/module/azure/database_account/_meta/data.json @@ -1,40 +1,38 @@ { - "@timestamp" : "2020-02-25T08:53:00.000Z", - "cloud" : { - "provider" : "azure", - "region" : "westeurope" - }, - "event" : { - "module" : "azure", - "duration" : 4877063600, - "dataset" : "azure.database_account" - }, - "metricset" : { - "name" : "database_account", - "period" : 300000 - }, - "azure" : { - "timegrain" : "PT5M", - "dimensions" : { - "database_name" : "testdb" - }, - "database_account" : { - "provisioned_throughput" : { - "max" : 400 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "database_account": { + "service_availability": { + "avg": 100 } }, - "resource" : { - "group" : "obs-test", - "tags" : { - "defaultExperience" : "Core (SQL)" + "namespace": "Microsoft.DocumentDb/databaseAccounts", + "resource": { + "group": "obs-infrastructure", + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", + "name": "obsaccount", + "tags": { + "defaultExperience": "Core (SQL)" }, - "name" : "obsaccount", - "type" : "Microsoft.DocumentDb/databaseAccounts" + "type": "Microsoft.DocumentDb/databaseAccounts" }, - "subscription_id" : "123456-qwer-1234-5678-12345678", - "namespace" : "Microsoft.DocumentDb/databaseAccounts" + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "timegrain": "PT1H" + }, + "cloud": { + "provider": "azure", + "region": "westeurope" + }, + "event": { + "dataset": "azure.database_account", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "database_account", + "period": 10000 }, - "service" : { - "type" : "azure" + "service": { + "type": "azure" } } diff --git a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go index 6fa35ee46984..4c00bd03af04 100644 --- a/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go +++ b/x-pack/metricbeat/module/azure/database_account/database_account_integration_test.go @@ -15,6 +15,9 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/monitor" ) func TestFetchMetricset(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index 4d2dcd73f6e8..f6f54c300e03 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -9,7 +9,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-03-01/resources" "github.com/stretchr/testify/mock" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -64,14 +63,3 @@ func (reporter *MockReporterV2) Error(err error) bool { args := reporter.Called(err) return args.Get(0).(bool) } - -// NewMockClient instantiates a new client with the mock azure service -func NewMockClient() *Client { - azureMockService := new(MockService) - client := &Client{ - AzureMonitorService: azureMockService, - Config: Config{}, - Log: logp.NewLogger("test azure monitor"), - } - return client -} diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml index cc106a3c59b1..d7e4831e2035 100644 --- a/x-pack/metricbeat/module/azure/module.yml +++ b/x-pack/metricbeat/module/azure/module.yml @@ -4,3 +4,4 @@ metricsets: - container_instance - container_service - database_account + - compute_vm diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/data.json b/x-pack/metricbeat/module/azure/monitor/_meta/data.json index 67f5b91b62be..16d7f82b8d89 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/data.json +++ b/x-pack/metricbeat/module/azure/monitor/_meta/data.json @@ -3,31 +3,29 @@ "azure": { "metrics": { "data_usage": { - "total": 0 + "total": 131072 }, "document_count": { - "total": 0 + "total": 2 }, "document_quota": { - "total": 53687091200 + "total": 107374182400 } }, "namespace": "Microsoft.DocumentDb/databaseAccounts", "resource": { "group": "obs-infrastructure", + "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", + "name": "obsaccount", "tags": { "defaultExperience": "Core (SQL)" }, "type": "Microsoft.DocumentDb/databaseAccounts" }, - "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", + "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", "timegrain": "PT5M" }, "cloud": { - "instance": { - "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.DocumentDb/databaseAccounts/obsaccount", - "name": "obsaccount" - }, "provider": "azure", "region": "westeurope" }, diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 4d00bcd64de8..82875f46de51 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -16,6 +16,8 @@ import ( "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" ) +const missingNamespace = "no metric definitions were found for resource %s and namespace %s. Verify if the namespace is spelled correctly or if it is supported by the resource in case." + // mapMetrics should validate and map the metric related configuration to relevant azure monitor api parameters func mapMetrics(client *azure.Client, resources []resources.GenericResource, resourceConfig azure.ResourceConfig) ([]azure.Metric, error) { var metrics []azure.Metric @@ -27,7 +29,11 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res return nil, errors.Wrapf(err, "no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) } if len(*metricDefinitions.Value) == 0 { - return nil, errors.Errorf("no metric definitions were found for resource %s and namespace %s.", *resource.ID, metric.Namespace) + if metric.IgnoreUnsupported { + client.Log.Infof(missingNamespace, *resource.ID, metric.Namespace) + continue + } + return nil, errors.Errorf(missingNamespace, *resource.ID, metric.Namespace) } // validate metric names and filter on the supported metrics @@ -54,7 +60,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res for _, metricName := range metricGroup { metricNames = append(metricNames, *metricName.Name.Value) } - metrics = append(metrics, client.CreateMetric(*resource.ID, resource, "", metric.Namespace, metricNames, key, dim, metric.Timegrain)) + metrics = append(metrics, client.CreateMetric(*resource.ID, "", metric.Namespace, metricNames, key, dim, metric.Timegrain)) } } } diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index 54b18aeaf73b..a15ee0089b9d 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -79,10 +79,7 @@ func TestMapMetric(t *testing.T) { resourceConfig.Metrics = []azure.MetricConfig{metricConfig} metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NoError(t, err) - assert.Equal(t, metrics[0].Resource.Id, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].ResourceId, "123") assert.Equal(t, metrics[0].Namespace, "namespace") assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity", "BytesRead"}) assert.Equal(t, metrics[0].Aggregations, "Average") @@ -100,10 +97,7 @@ func TestMapMetric(t *testing.T) { assert.NoError(t, err) assert.True(t, len(metrics) > 0) - assert.Equal(t, metrics[0].Resource.Id, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].ResourceId, "123") assert.Equal(t, metrics[0].Namespace, "namespace") assert.Equal(t, metrics[0].Names, []string{"TotalRequests", "Capacity"}) assert.Equal(t, metrics[0].Aggregations, "Average") @@ -144,7 +138,6 @@ func TestIntersections(t *testing.T) { intersection, difference = intersections(firstStr, sercondStr) assert.Equal(t, len(intersection), 0) assert.Equal(t, difference, []string{"test4", "test5"}) - } func TestGetMetricDefinitionsByNames(t *testing.T) { diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index 4657e3bc43dd..053da3db05ba 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -124,3 +124,24 @@ func (service *MonitorService) GetMetricValues(resourceId string, namespace stri } return metrics, interval, nil } + +// getResourceNameFormId maps resource group from resource ID +func getResourceNameFromId(path string) string { + params := strings.Split(path, "/") + if strings.HasSuffix(path, "/") { + return params[len(params)-2] + } + return params[len(params)-1] + +} + +// getResourceTypeFromId maps resource group from resource ID +func getResourceTypeFromId(path string) string { + params := strings.Split(path, "/") + for i, param := range params { + if param == "providers" { + return fmt.Sprintf("%s/%s", params[i+1], params[i+2]) + } + } + return "" +} diff --git a/x-pack/metricbeat/module/azure/monitor_service_test.go b/x-pack/metricbeat/module/azure/monitor_service_test.go new file mode 100644 index 000000000000..7855f554e1e4 --- /dev/null +++ b/x-pack/metricbeat/module/azure/monitor_service_test.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetResourceTypeFromID(t *testing.T) { + path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" + rType := getResourceTypeFromId(path) + assert.Equal(t, rType, "Microsoft.Compute/virtualMachines") +} + +func TestGetResourceNameFromID(t *testing.T) { + path := "subscriptions/qw3e45r6t-23ws-1234-6587-1234ed4532/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak" + name := getResourceNameFromId(path) + assert.Equal(t, name, "obstestmemleak") +} diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index c55757d24acc..ba2c93b46186 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -11,27 +11,32 @@ import ( // Resource will contain the main azure resource details type Resource struct { - // SubId is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms - SubId string Id string Name string Location string - Type string Group string Tags map[string]string Subscription string - Size string + Type string + Vm VmResource +} + +type VmResource struct { + Size string + Id string } // Metric will contain the main azure metric details type Metric struct { - Resource Resource Namespace string Names []string Aggregations string Dimensions []Dimension Values []MetricValue TimeGrain string + ResourceId string + // ResourceSubId is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms + ResourceSubId string } // Dimension represents the azure metric dimension details diff --git a/x-pack/metricbeat/module/azure/storage/client_helper.go b/x-pack/metricbeat/module/azure/storage/client_helper.go index 3fa8bb7e4b9e..44e49831790b 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper.go @@ -61,7 +61,7 @@ func mapMetrics(client *azure.Client, resources []resources.GenericResource, res if dimension != azure.NoDimension { dimensions = []azure.Dimension{{Name: dimension, Value: "*"}} } - metrics = append(metrics, client.MapMetricByPrimaryAggregation(mets, resource, resourceID, "", namespace, dimensions, time)...) + metrics = append(metrics, client.MapMetricByPrimaryAggregation(mets, *resource.ID, resourceID, namespace, dimensions, time)...) } } } diff --git a/x-pack/metricbeat/module/azure/storage/client_helper_test.go b/x-pack/metricbeat/module/azure/storage/client_helper_test.go index 9f18085ef0ca..0aab2a2395f1 100644 --- a/x-pack/metricbeat/module/azure/storage/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/storage/client_helper_test.go @@ -109,15 +109,9 @@ func TestMapMetric(t *testing.T) { client.AzureMonitorService = m metrics, err := mapMetrics(client, []resources.GenericResource{resource}, resourceConfig) assert.NoError(t, err) - assert.Equal(t, metrics[0].Resource.Id, "123") - assert.Equal(t, metrics[0].Resource.Name, "resourceName") - assert.Equal(t, metrics[0].Resource.Type, "resourceType") - assert.Equal(t, metrics[0].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[0].ResourceId, "123") assert.Equal(t, metrics[0].Namespace, "Microsoft.Storage/storageAccounts") - assert.Equal(t, metrics[1].Resource.Id, "123") - assert.Equal(t, metrics[1].Resource.Name, "resourceName") - assert.Equal(t, metrics[1].Resource.Type, "resourceType") - assert.Equal(t, metrics[1].Resource.Location, "resourceLocation") + assert.Equal(t, metrics[1].ResourceId, "123") assert.Equal(t, metrics[1].Namespace, "Microsoft.Storage/storageAccounts") assert.Equal(t, metrics[0].Dimensions, []azure.Dimension(nil)) assert.Equal(t, metrics[1].Dimensions, []azure.Dimension(nil))