diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a96a23db9d8..d95ad15bcaf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -729,6 +729,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124] - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255] - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137] +- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381] *Packetbeat* diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index 9cd3eba55b8..3028d531c1e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{ azHeaders := map[string]string{"Metadata": "true"} azSchema := func(m map[string]interface{}) common.MapStr { out, _ := s.Schema{ + "account": s.Object{ + "id": c.Str("subscriptionId"), + }, "instance": s.Object{ "id": c.Str("vmId"), "name": c.Str("name"), diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 307ac60abad..a988cc8873f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{ "sku": "14.04.4-LTS", "version": "14.04.201605091", "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "vmSize": "Standard_D3_v2" + "vmSize": "Standard_D3_v2", + "subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e" }` func initAzureTestServer() *httptest.Server { @@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) { "machine": common.MapStr{ "type": "Standard_D3_v2", }, + "account": common.MapStr{ + "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e", + }, "region": "eastus2", }, } diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go index ba8f35c7db6..1342170aec0 100644 --- a/x-pack/metricbeat/module/azure/add_metadata.go +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) { } } -func addCloudVMMetadata(event *mb.Event, vm VmResource) { +func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) { if vm.Name != "" { event.RootFields.Put("cloud.instance.name", vm.Name) event.RootFields.Put("host.name", vm.Name) @@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) { if vm.Size != "" { event.RootFields.Put("cloud.machine.type", vm.Size) } + if subscriptionId != "" { + event.RootFields.Put("cloud.account.id", subscriptionId) + } } diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 1da5cfb63ab..c38407c76f1 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,62 +2,86 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { + "disk_read_bytes": { + "total": 0 + }, "disk_read_operations_per_sec": { - "avg": 3.3875 + "avg": 0 + }, + "disk_write_bytes": { + "total": 2969709.4 }, "disk_write_operations_per_sec": { - "avg": 0.6705 + "avg": 0.7809999999999999 }, "inbound_flows": { - "avg": 28.4 + "avg": 10 }, "inbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 + }, + "network_in": { + "total": 1478232 + }, + "network_in_total": { + "total": 1569665 + }, + "network_out": { + "total": 793344 + }, + "network_out_total": { + "total": 1074624 + }, + "os_disk_bandwidth_consumed_percentage": { + "avg": 0 + }, + "os_disk_iops_consumed_percentage": { + "avg": 0 }, "os_disk_queue_depth": { - "avg": 0.00125 + "avg": 0.002 }, "os_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "os_per_disk_qd": { - "avg": 0.00125 + "avg": 0.002 }, "os_per_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_per_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_per_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "outbound_flows": { - "avg": 28.4 + "avg": 10 }, "outbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 }, "per_disk_qd": { - "avg": 0.0025 + "avg": 0 }, "per_disk_read_bytes_per_sec": { - "avg": 51985.035 + "avg": 0 }, "per_disk_read_operations_per_sec": { - "avg": 2.92875 + "avg": 0 }, "per_disk_write_bytes_per_sec": { "avg": 0 @@ -66,26 +90,41 @@ "avg": 0 }, "percentage_cpu": { - "avg": 9.747 + "avg": 1.8719999999999999 + }, + "vm_cached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_cached_iops_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_iops_consumed_percentage": { + "avg": 0 } }, "namespace": "Microsoft.Compute/virtualMachines", "resource": { "group": "obs-infrastructure", - "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux", - "name": "obslinux", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz", + "name": "testaz", "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz" }, "machine": { - "type": "Basic_A0" + "type": "Standard_A1_v2" }, "provider": "azure", "region": "westeurope" @@ -97,10 +136,28 @@ }, "host": { "cpu": { - "pct": 0.09747 + "pct": 0.01872 }, - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 2969709.4 + } + }, + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz", + "network": { + "in": { + "bytes": 1569665, + "packets": 1478232 + }, + "out": { + "bytes": 1074624, + "packets": 793344 + } + } }, "metricset": { "name": "compute_vm", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index e8f59859d8b..2d99d8ff6b0 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -2,6 +2,12 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm_scaleset": { + "cpu_credits_consumed": { + "avg": 0.006999999999999999 + }, + "cpu_credits_remaining": { + "avg": 84.72 + }, "os_per_disk_qd": { "avg": 0 }, @@ -12,35 +18,34 @@ "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 1872.1200000000001 + "avg": 3846.531 }, "os_per_disk_write_operations_per_sec": { - "avg": 0.296 + "avg": 0.5519999999999999 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { - "group": "testgroup", - "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset", - "name": "vmscaleset", - "tags": { - "environment": "staging", - "role": "allocator" - }, + "group": "obs-infrastructure", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss", "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "machine": { - "type": "Standard_D4s_v3" + "type": "Standard_B1ls" }, "provider": "azure", - "region": "eastus2" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm_scaleset", @@ -48,7 +53,7 @@ "module": "azure" }, "host": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "metricset": { "name": "compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index bf77f657416..4bbb59a4606 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -82,7 +82,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupDimValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } } @@ -90,7 +90,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupTimeValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } if client.Config.DefaultResourceType == "" {