From bccee1fa1a41099610d7e19695346e285a2b287f Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 21 Aug 2020 07:12:18 -0600 Subject: [PATCH] Cherry-pick #20641 to 7.x: Add host inventory metrics to azure compute_vm metricset (#20725) * Add host inventory metrics to azure compute_vm metricset (#20641) (cherry picked from commit 6bf61cb688c095e159584bbaa2528770539f703b) --- CHANGELOG.next.asciidoc | 1 + .../module/azure/compute_vm/_meta/data.json | 221 ++++++++++-------- x-pack/metricbeat/module/azure/data.go | 31 +++ 3 files changed, 156 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 52e7ed1c942a..31aae20d241e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -724,6 +724,7 @@ field. You can revert this change by configuring tags for the module and omittin - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] - Add host inventory metrics into aws ec2 metricset. {pull}20171[20171] - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] +- Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641] *Packetbeat* diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 960eb88cf7e4..1665af3d455f 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -1,131 +1,158 @@ { - "@timestamp":"2020-08-05T10:36:00.000Z", - "cloud":{ - "provider":"azure", - "region":"westeurope", - "instance":{ - "id":"/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obstestmemleak", - "name":"obstestmemleak" - }, - "machine":{ - "type":"Standard_B2ms" - } - }, - "event":{ - "dataset":"azure.compute_vm", - "module":"azure", - "duration":5364973100 - }, - "metricset":{ - "name":"compute_vm", - "period":300000 - }, - "azure":{ - "timegrain":"PT5M", - "resource":{ - "type":"Microsoft.Compute/virtualMachines", - "group":"obs-infrastructure", - "tags":{ - "vmtest":"value1, value 2", - "vmtest1":"value3" - } - }, - "subscription_id":"fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", - "namespace":"Microsoft.Compute/virtualMachines", - "compute_vm":{ - "outbound_flows":{ - "avg":291.2 + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "compute_vm": { + "cpu_credits_consumed": { + "avg": 0.01 + }, + "cpu_credits_remaining": { + "avg": 288 + }, + "disk_read_bytes": { + "total": 4019543.24 }, - "outbound_flows_maximum_creation_rate":{ - "avg":6 + "disk_read_operations_per_sec": { + "avg": 0.0605 }, - "inbound_flows_maximum_creation_rate":{ - "avg":6 + "disk_write_bytes": { + "total": 16739967.27 }, - "os_per_disk_read_bytes_per_sec":{ - "avg":341.35875 + "disk_write_operations_per_sec": { + "avg": 1.9525 }, - "per_disk_read_bytes_per_sec":{ - "avg":0 + "inbound_flows": { + "avg": 31 }, - "os_disk_queue_depth":{ - "avg":0 + "inbound_flows_maximum_creation_rate": { + "avg": 0.8 }, - "os_per_disk_qd":{ - "avg":0 + "network_in": { + "total": 1501534 }, - "os_disk_read_bytes_per_sec":{ - "avg":341.35875 + "network_in_total": { + "total": 1806936 }, - "network_in":{ - "total":3933481.0 + "network_out": { + "total": 1647640 }, - "network_out":{ - "total":7169752.0 + "network_out_total": { + "total": 3633130 }, - "os_disk_write_operations_per_sec":{ - "avg":4.994285714285715 + "os_disk_queue_depth": { + "avg": 0 }, - "percentage_cpu":{ - "avg":8.780625 + "os_disk_read_bytes_per_sec": { + "avg": 13398.475 }, - "cpu_credits_remaining":{ - "avg":864 + "os_disk_read_operations_per_sec": { + "avg": 0.121 }, - "network_out_total":{ - "total":9249694.0 + "os_disk_write_bytes_per_sec": { + "avg": 55799.886 }, - "disk_read_operations_per_sec":{ - "avg":0.022857142857142857 + "os_disk_write_operations_per_sec": { + "avg": 3.905 }, - "disk_write_bytes":{ - "total":1.759878456E7 + "os_per_disk_qd": { + "avg": 0 }, - "os_disk_write_bytes_per_sec":{ - "avg":73050.70375 + "os_per_disk_read_bytes_per_sec": { + "avg": 13398.475 }, - "cpu_credits_consumed":{ - "avg":0.175 + "os_per_disk_read_operations_per_sec": { + "avg": 0.121 }, - "per_disk_qd":{ - "avg":0 + "os_per_disk_write_bytes_per_sec": { + "avg": 55799.886 }, - "disk_write_operations_per_sec":{ - "avg":2.5221428571428572 + "os_per_disk_write_operations_per_sec": { + "avg": 3.905 }, - "os_per_disk_write_operations_per_sec":{ - "avg":4.994285714285715 + "outbound_flows": { + "avg": 31 }, - "per_disk_write_operations_per_sec":{ - "avg":0.049999999999999996 + "outbound_flows_maximum_creation_rate": { + "avg": 0.8 }, - "inbound_flows":{ - "avg":291.2 + "per_disk_qd": { + "avg": 0 }, - "os_disk_read_operations_per_sec":{ - "avg":0.045714285714285714 + "per_disk_read_bytes_per_sec": { + "avg": 0 }, - "disk_read_bytes":{ - "total":81926.52 + "per_disk_read_operations_per_sec": { + "avg": 0 }, - "network_in_total":{ - "total":4920267.0 + "per_disk_write_bytes_per_sec": { + "avg": 0 }, - "per_disk_read_operations_per_sec":{ - "avg":0 + "per_disk_write_operations_per_sec": { + "avg": 0 }, - "per_disk_write_bytes_per_sec":{ - "avg":277.56 + "percentage_cpu": { + "avg": 1.585 }, - "os_per_disk_write_bytes_per_sec":{ - "avg":73050.70375 + "premium_os_disk_cache_read_hit": { + "avg": 100 + }, + "premium_os_disk_cache_read_miss": { + "avg": 0 + } + }, + "namespace": "Microsoft.Compute/virtualMachines", + "resource": { + "group": "obs-test", + "type": "Microsoft.Compute/virtualMachines" + }, + "subscription_id": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "timegrain": "PT5M" + }, + "cloud": { + "instance": { + "id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test", + "name": "perfmon-test" + }, + "machine": { + "type": "Standard_B1ms" + }, + "provider": "azure", + "region": "westeurope" + }, + "event": { + "dataset": "azure.compute_vm", + "duration": 115000, + "module": "azure" + }, + "host": { + "cpu": { + "pct": 0.01585 + }, + "disk": { + "read": { + "bytes": 4019543.24 }, - "os_per_disk_read_operations_per_sec":{ - "avg":0.045714285714285714 + "write": { + "bytes": 16739967.27 + } + }, + "id": "/subscriptions/7657426d-c4c3-44ac-88a2-3b2cd59e6dba/resourceGroups/obs-test/providers/Microsoft.Compute/virtualMachines/perfmon-test", + "name": "perfmon-test", + "network": { + "in": { + "bytes": 1806936, + "packets": 1501534 + }, + "out": { + "bytes": 3633130, + "packets": 1647640 } } }, - "service":{ - "type":"azure" + "metricset": { + "name": "compute_vm", + "period": 10000 + }, + "service": { + "type": "azure" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index 23a043ae6f97..936a4887f713 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -173,10 +173,13 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) event.RootFields.Put("cloud.provider", "azure") event.RootFields.Put("cloud.region", metric.Resource.Location) event.RootFields.Put("cloud.instance.name", metric.Resource.Name) + event.RootFields.Put("host.name", metric.Resource.Name) if metric.Resource.SubId != "" { event.RootFields.Put("cloud.instance.id", metric.Resource.SubId) + event.RootFields.Put("host.id", metric.Resource.SubId) } else { event.RootFields.Put("cloud.instance.id", metric.Resource.Id) + event.RootFields.Put("host.id", metric.Resource.Id) } if metric.Resource.Size != "" { event.RootFields.Put("cloud.machine.type", metric.Resource.Size) @@ -200,6 +203,7 @@ func createEvent(timestamp time.Time, metric Metric, metricValues []MetricValue) metricList.Put(fmt.Sprintf("%s.%s", metricNameString, "count"), *value.count) } } + event = addHostFields(event, metricList) return event, metricList } @@ -229,3 +233,30 @@ func returnAllDimensions(dimensions []Dimension) (bool, []Dimension) { } return true, dims } + +func addHostFields(event mb.Event, metricList common.MapStr) mb.Event { + hostFieldTable := map[string]string{ + "percentage_cpu.avg": "host.cpu.pct", + "network_in_total.total": "host.network.in.bytes", + "network_in.total": "host.network.in.packets", + "network_out_total.total": "host.network.out.bytes", + "network_out.total": "host.network.out.packets", + "disk_read_bytes.total": "host.disk.read.bytes", + "disk_write_bytes.total": "host.disk.write.bytes", + } + + for metricName, hostName := range hostFieldTable { + metricValue, err := metricList.GetValue(metricName) + if err != nil { + continue + } + + if value, ok := metricValue.(float64); ok { + if metricName == "percentage_cpu.avg" { + value = value / 100 + } + event.RootFields.Put(hostName, value) + } + } + return event +}