Map cloud.account.id to azure sub id (elastic#21483)

* mofidy doc * tet * add cloud * changelog
narph · Oct 5, 2020 · 9aab0c0 · 9aab0c0
1 parent 67a2d38
commit 9aab0c0
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 47 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -731,6 +731,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 - Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
+- Map cloud data filed `cloud.account.id` to azure subscription.  {pull}21483[21483] {issue}21381[21381]
 
 *Packetbeat*
 

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go
@@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{
 		azHeaders := map[string]string{"Metadata": "true"}
 		azSchema := func(m map[string]interface{}) common.MapStr {
 			out, _ := s.Schema{
+				"account": s.Object{
+					"id": c.Str("subscriptionId"),
+				},
 				"instance": s.Object{
 					"id":   c.Str("vmId"),
 					"name": c.Str("name"),

diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go
@@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{
 	"sku": "14.04.4-LTS",
 	"version": "14.04.201605091",
 	"vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e",
-	"vmSize": "Standard_D3_v2"
+	"vmSize": "Standard_D3_v2",
+	"subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e"
 }`
 
 func initAzureTestServer() *httptest.Server {
@@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) {
 			"machine": common.MapStr{
 				"type": "Standard_D3_v2",
 			},
+			"account": common.MapStr{
+				"id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e",
+			},
 			"region": "eastus2",
 		},
 	}

diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go
@@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) {
 	}
 }
 
-func addCloudVMMetadata(event *mb.Event, vm VmResource) {
+func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) {
 	if vm.Name != "" {
 		event.RootFields.Put("cloud.instance.name", vm.Name)
 		event.RootFields.Put("host.name", vm.Name)
@@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) {
 	if vm.Size != "" {
 		event.RootFields.Put("cloud.machine.type", vm.Size)
 	}
+	if subscriptionId != "" {
+		event.RootFields.Put("cloud.account.id", subscriptionId)
+	}
 }
diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json
@@ -2,62 +2,86 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "azure": {
         "compute_vm": {
+            "disk_read_bytes": {
+                "total": 0
+            },
             "disk_read_operations_per_sec": {
-                "avg": 3.3875
+                "avg": 0
+            },
+            "disk_write_bytes": {
+                "total": 2969709.4
             },
             "disk_write_operations_per_sec": {
-                "avg": 0.6705
+                "avg": 0.7809999999999999
             },
             "inbound_flows": {
-                "avg": 28.4
+                "avg": 10
             },
             "inbound_flows_maximum_creation_rate": {
-                "avg": 10.4
+                "avg": 10.6
+            },
+            "network_in": {
+                "total": 1478232
+            },
+            "network_in_total": {
+                "total": 1569665
+            },
+            "network_out": {
+                "total": 793344
+            },
+            "network_out_total": {
+                "total": 1074624
+            },
+            "os_disk_bandwidth_consumed_percentage": {
+                "avg": 0
+            },
+            "os_disk_iops_consumed_percentage": {
+                "avg": 0
             },
             "os_disk_queue_depth": {
-                "avg": 0.00125
+                "avg": 0.002
             },
             "os_disk_read_bytes_per_sec": {
-                "avg": 602589.1825
+                "avg": 0
             },
             "os_disk_read_operations_per_sec": {
-                "avg": 5.28375
+                "avg": 0
             },
             "os_disk_write_bytes_per_sec": {
-                "avg": 14137.59375
+                "avg": 9899.025
             },
             "os_disk_write_operations_per_sec": {
-                "avg": 1.46875
+                "avg": 1.5619999999999998
             },
             "os_per_disk_qd": {
-                "avg": 0.00125
+                "avg": 0.002
             },
             "os_per_disk_read_bytes_per_sec": {
-                "avg": 602589.1825
+                "avg": 0
             },
             "os_per_disk_read_operations_per_sec": {
-                "avg": 5.28375
+                "avg": 0
             },
             "os_per_disk_write_bytes_per_sec": {
-                "avg": 14137.59375
+                "avg": 9899.025
             },
             "os_per_disk_write_operations_per_sec": {
-                "avg": 1.46875
+                "avg": 1.5619999999999998
             },
             "outbound_flows": {
-                "avg": 28.4
+                "avg": 10
             },
             "outbound_flows_maximum_creation_rate": {
-                "avg": 10.4
+                "avg": 10.6
             },
             "per_disk_qd": {
-                "avg": 0.0025
+                "avg": 0
             },
             "per_disk_read_bytes_per_sec": {
-                "avg": 51985.035
+                "avg": 0
             },
             "per_disk_read_operations_per_sec": {
-                "avg": 2.92875
+                "avg": 0
             },
             "per_disk_write_bytes_per_sec": {
                 "avg": 0
@@ -66,26 +90,41 @@
                 "avg": 0
             },
             "percentage_cpu": {
-                "avg": 9.747
+                "avg": 1.8719999999999999
+            },
+            "vm_cached_bandwidth_consumed_percentage": {
+                "avg": 0
+            },
+            "vm_cached_iops_consumed_percentage": {
+                "avg": 0
+            },
+            "vm_uncached_bandwidth_consumed_percentage": {
+                "avg": 0
+            },
+            "vm_uncached_iops_consumed_percentage": {
+                "avg": 0
             }
         },
         "namespace": "Microsoft.Compute/virtualMachines",
         "resource": {
             "group": "obs-infrastructure",
-            "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux",
-            "name": "obslinux",
+            "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz",
+            "name": "testaz",
             "type": "Microsoft.Compute/virtualMachines"
         },
-        "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364",
+        "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6",
         "timegrain": "PT5M"
     },
     "cloud": {
+        "account": {
+            "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6"
+        },
         "instance": {
-            "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0",
-            "name": "obslinux"
+            "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad",
+            "name": "testaz"
         },
         "machine": {
-            "type": "Basic_A0"
+            "type": "Standard_A1_v2"
         },
         "provider": "azure",
         "region": "westeurope"
@@ -97,10 +136,28 @@
     },
     "host": {
         "cpu": {
-            "pct": 0.09747
+            "pct": 0.01872
         },
-        "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0",
-        "name": "obslinux"
+        "disk": {
+            "read": {
+                "bytes": 0
+            },
+            "write": {
+                "bytes": 2969709.4
+            }
+        },
+        "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad",
+        "name": "testaz",
+        "network": {
+            "in": {
+                "bytes": 1569665,
+                "packets": 1478232
+            },
+            "out": {
+                "bytes": 1074624,
+                "packets": 793344
+            }
+        }
     },
     "metricset": {
         "name": "compute_vm",

diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json
@@ -2,6 +2,12 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "azure": {
         "compute_vm_scaleset": {
+            "cpu_credits_consumed": {
+                "avg": 0.006999999999999999
+            },
+            "cpu_credits_remaining": {
+                "avg": 84.72
+            },
             "os_per_disk_qd": {
                 "avg": 0
             },
@@ -12,43 +18,42 @@
                 "avg": 0
             },
             "os_per_disk_write_bytes_per_sec": {
-                "avg": 1872.1200000000001
+                "avg": 3846.531
             },
             "os_per_disk_write_operations_per_sec": {
-                "avg": 0.296
+                "avg": 0.5519999999999999
             }
         },
         "namespace": "Microsoft.Compute/virtualMachineScaleSets",
         "resource": {
-            "group": "testgroup",
-            "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset",
-            "name": "vmscaleset",
-            "tags": {
-                "environment": "staging",
-                "role": "allocator"
-            },
+            "group": "obs-infrastructure",
+            "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss",
+            "name": "obslinuxvmss",
             "type": "Microsoft.Compute/virtualMachineScaleSets"
         },
-        "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364",
+        "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6",
         "timegrain": "PT5M"
     },
     "cloud": {
+        "account": {
+            "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6"
+        },
         "instance": {
-            "name": "vmscaleset"
+            "name": "obslinuxvmss"
         },
         "machine": {
-            "type": "Standard_D4s_v3"
+            "type": "Standard_B1ls"
         },
         "provider": "azure",
-        "region": "eastus2"
+        "region": "westeurope"
     },
     "event": {
         "dataset": "azure.compute_vm_scaleset",
         "duration": 115000,
         "module": "azure"
     },
     "host": {
-        "name": "vmscaleset"
+        "name": "obslinuxvmss"
     },
     "metricset": {
         "name": "compute_vm_scaleset",

diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go
@@ -82,15 +82,15 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error
 						event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues)
 						if client.Config.AddCloudMetadata {
 							vm = client.GetVMForMetaData(&resource, groupDimValues)
-							addCloudVMMetadata(&event, vm)
+							addCloudVMMetadata(&event, vm, resource.Subscription)
 						}
 					}
 				}
 			} else {
 				event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues)
 				if client.Config.AddCloudMetadata {
 					vm = client.GetVMForMetaData(&resource, groupTimeValues)
-					addCloudVMMetadata(&event, vm)
+					addCloudVMMetadata(&event, vm, resource.Subscription)
 				}
 			}
 			if client.Config.DefaultResourceType == "" {