From 938e66ccf352e2925b99fc988b28c46fc3ee304e Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 23 Jul 2020 11:01:08 +0200 Subject: [PATCH 1/4] mofidy doc --- .../module/azure/app_insights/_meta/docs.asciidoc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc index 2ba1150078b..2b587acbbdd 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc @@ -45,17 +45,15 @@ This value is only valid when segment is specified. `filter`:: (_string_) An expression used to filter the results. This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving. -Users can select the options to retrieve all metrics from a specific namespace using the following: +Example configuration: ["source","yaml"] ---- - metrics: - - id: ["*"] - timespan: "Microsoft.Storage/storageAccounts" +metrics: + - id: ["requests/count", "requests/failed"] + segment: "request/name" + aggregation: ["sum"] ---- -A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. - - From 6e5b67f0a3f7602e97dfc554206514286a47c1e8 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 2 Oct 2020 16:04:13 +0200 Subject: [PATCH 2/4] tet --- x-pack/metricbeat/module/azure/resources.go | 1 + .../modules.d/{azure.yml.disabled => azure.yml} | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) rename x-pack/metricbeat/modules.d/{azure.yml.disabled => azure.yml} (88%) diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index 0a723c82bd5..edfc1f220ef 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -27,6 +27,7 @@ type VmResource struct { Size string Id string Name string + AZ string } // Metric will contain the main azure metric details diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml similarity index 88% rename from x-pack/metricbeat/modules.d/azure.yml.disabled rename to x-pack/metricbeat/modules.d/azure.yml index 23211f47206..683c8b4fc11 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml @@ -1,18 +1,7 @@ # Module: azure # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-azure.html -- module: azure - metricsets: - - monitor - enabled: true - period: 300s - client_id: '${AZURE_CLIENT_ID:""}' - client_secret: '${AZURE_CLIENT_SECRET:""}' - tenant_id: '${AZURE_TENANT_ID:""}' - subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' - refresh_list_interval: 600s - resources: -# - resource_query: "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'" + # metrics: # - name: ["DataUsage", "DocumentCount", "DocumentQuota"] # namespace: "Microsoft.DocumentDb/databaseAccounts" From d9aa5d612e6ed5c5874605f6e6d7ecddb49c6fb6 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 2 Oct 2020 17:34:28 +0200 Subject: [PATCH 3/4] add cloud --- CHANGELOG.next.asciidoc | 1 + .../add_cloud_metadata/provider_azure_vm.go | 3 + .../provider_azure_vm_test.go | 6 +- .../metricbeat/module/azure/add_metadata.go | 5 +- .../module/azure/compute_vm/_meta/data.json | 115 +++++++++++++----- .../azure/compute_vm_scaleset/_meta/data.json | 33 ++--- x-pack/metricbeat/module/azure/data.go | 4 +- .../{azure.yml => azure.yml.disabled} | 13 +- 8 files changed, 132 insertions(+), 48 deletions(-) rename x-pack/metricbeat/modules.d/{azure.yml => azure.yml.disabled} (88%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a96a23db9d8..4d397580fc9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -729,6 +729,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] +- Map cloud data filed `cloud.account.id` to azure subscription. *Packetbeat* diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index 9cd3eba55b8..3028d531c1e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{ azHeaders := map[string]string{"Metadata": "true"} azSchema := func(m map[string]interface{}) common.MapStr { out, _ := s.Schema{ + "account": s.Object{ + "id": c.Str("subscriptionId"), + }, "instance": s.Object{ "id": c.Str("vmId"), "name": c.Str("name"), diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 307ac60abad..a988cc8873f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{ "sku": "14.04.4-LTS", "version": "14.04.201605091", "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "vmSize": "Standard_D3_v2" + "vmSize": "Standard_D3_v2", + "subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e" }` func initAzureTestServer() *httptest.Server { @@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) { "machine": common.MapStr{ "type": "Standard_D3_v2", }, + "account": common.MapStr{ + "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e", + }, "region": "eastus2", }, } diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go index ba8f35c7db6..1342170aec0 100644 --- a/x-pack/metricbeat/module/azure/add_metadata.go +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) { } } -func addCloudVMMetadata(event *mb.Event, vm VmResource) { +func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) { if vm.Name != "" { event.RootFields.Put("cloud.instance.name", vm.Name) event.RootFields.Put("host.name", vm.Name) @@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) { if vm.Size != "" { event.RootFields.Put("cloud.machine.type", vm.Size) } + if subscriptionId != "" { + event.RootFields.Put("cloud.account.id", subscriptionId) + } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 1da5cfb63ab..c38407c76f1 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,62 +2,86 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { + "disk_read_bytes": { + "total": 0 + }, "disk_read_operations_per_sec": { - "avg": 3.3875 + "avg": 0 + }, + "disk_write_bytes": { + "total": 2969709.4 }, "disk_write_operations_per_sec": { - "avg": 0.6705 + "avg": 0.7809999999999999 }, "inbound_flows": { - "avg": 28.4 + "avg": 10 }, "inbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 + }, + "network_in": { + "total": 1478232 + }, + "network_in_total": { + "total": 1569665 + }, + "network_out": { + "total": 793344 + }, + "network_out_total": { + "total": 1074624 + }, + "os_disk_bandwidth_consumed_percentage": { + "avg": 0 + }, + "os_disk_iops_consumed_percentage": { + "avg": 0 }, "os_disk_queue_depth": { - "avg": 0.00125 + "avg": 0.002 }, "os_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "os_per_disk_qd": { - "avg": 0.00125 + "avg": 0.002 }, "os_per_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_per_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_per_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "outbound_flows": { - "avg": 28.4 + "avg": 10 }, "outbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 }, "per_disk_qd": { - "avg": 0.0025 + "avg": 0 }, "per_disk_read_bytes_per_sec": { - "avg": 51985.035 + "avg": 0 }, "per_disk_read_operations_per_sec": { - "avg": 2.92875 + "avg": 0 }, "per_disk_write_bytes_per_sec": { "avg": 0 @@ -66,26 +90,41 @@ "avg": 0 }, "percentage_cpu": { - "avg": 9.747 + "avg": 1.8719999999999999 + }, + "vm_cached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_cached_iops_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_iops_consumed_percentage": { + "avg": 0 } }, "namespace": "Microsoft.Compute/virtualMachines", "resource": { "group": "obs-infrastructure", - "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux", - "name": "obslinux", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz", + "name": "testaz", "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz" }, "machine": { - "type": "Basic_A0" + "type": "Standard_A1_v2" }, "provider": "azure", "region": "westeurope" @@ -97,10 +136,28 @@ }, "host": { "cpu": { - "pct": 0.09747 + "pct": 0.01872 }, - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 2969709.4 + } + }, + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz", + "network": { + "in": { + "bytes": 1569665, + "packets": 1478232 + }, + "out": { + "bytes": 1074624, + "packets": 793344 + } + } }, "metricset": { "name": "compute_vm", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index e8f59859d8b..2d99d8ff6b0 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -2,6 +2,12 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm_scaleset": { + "cpu_credits_consumed": { + "avg": 0.006999999999999999 + }, + "cpu_credits_remaining": { + "avg": 84.72 + }, "os_per_disk_qd": { "avg": 0 }, @@ -12,35 +18,34 @@ "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 1872.1200000000001 + "avg": 3846.531 }, "os_per_disk_write_operations_per_sec": { - "avg": 0.296 + "avg": 0.5519999999999999 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { - "group": "testgroup", - "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset", - "name": "vmscaleset", - "tags": { - "environment": "staging", - "role": "allocator" - }, + "group": "obs-infrastructure", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss", "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "machine": { - "type": "Standard_D4s_v3" + "type": "Standard_B1ls" }, "provider": "azure", - "region": "eastus2" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm_scaleset", @@ -48,7 +53,7 @@ "module": "azure" }, "host": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "metricset": { "name": "compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index bf77f657416..4bbb59a4606 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -82,7 +82,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupDimValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } } @@ -90,7 +90,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupTimeValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } if client.Config.DefaultResourceType == "" { diff --git a/x-pack/metricbeat/modules.d/azure.yml b/x-pack/metricbeat/modules.d/azure.yml.disabled similarity index 88% rename from x-pack/metricbeat/modules.d/azure.yml rename to x-pack/metricbeat/modules.d/azure.yml.disabled index 683c8b4fc11..23211f47206 100644 --- a/x-pack/metricbeat/modules.d/azure.yml +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -1,7 +1,18 @@ # Module: azure # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-azure.html - +- module: azure + metricsets: + - monitor + enabled: true + period: 300s + client_id: '${AZURE_CLIENT_ID:""}' + client_secret: '${AZURE_CLIENT_SECRET:""}' + tenant_id: '${AZURE_TENANT_ID:""}' + subscription_id: '${AZURE_SUBSCRIPTION_ID:""}' + refresh_list_interval: 600s + resources: +# - resource_query: "resourceType eq 'Microsoft.DocumentDb/databaseAccounts'" # metrics: # - name: ["DataUsage", "DocumentCount", "DocumentQuota"] # namespace: "Microsoft.DocumentDb/databaseAccounts" From ef03efb33d82cbe20c57382c7f474377f2521e55 Mon Sep 17 00:00:00 2001 From: Mariana Date: Fri, 2 Oct 2020 17:38:06 +0200 Subject: [PATCH 4/4] changelog --- CHANGELOG.next.asciidoc | 2 +- x-pack/metricbeat/module/azure/resources.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4d397580fc9..d95ad15bcaf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -729,7 +729,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] -- Map cloud data filed `cloud.account.id` to azure subscription. +- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] *Packetbeat* diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index edfc1f220ef..0a723c82bd5 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -27,7 +27,6 @@ type VmResource struct { Size string Id string Name string - AZ string } // Metric will contain the main azure metric details