Trim project from GCE cloud metadata (#10987)

This removes the project info from the cloud.machine.type and cloud.availability_zone fields that are added to the add_cloud_metadata processor for Google Compute Engine (GCE). Fixes #10968, #10775 (docs)
elastic · Mar 1, 2019 · f527cd5 · f527cd5
1 parent 76a2377
commit f527cd5
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 54 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -31,6 +31,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow Central Management to send events back to kibana. {issue}9382[9382]
 - Initialize the Paths before the keystore and save the keystore into `data/{beatname}.keystore`. {pull}10706[10706]
 - Add `cleanup_timeout` option to docker autodiscover, to wait some time before removing configurations after a container is stopped. {issue]10374[10374] {pull}10905[10905]
+- On Google Cloud Engine (GCE) the add_cloud_metadata will now trim the project
+  info from the cloud.machine.type and cloud.availability_zone. {issue}10968[10968]
 
 *Auditbeat*
 

diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc
@@ -522,14 +522,12 @@ _EC2_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "availability_zone": "us-east-1c",
-      "instance_id": "i-4e123456",
-      "machine_type": "t2.medium",
-      "provider": "ec2",
-      "region": "us-east-1"
-    }
+  "cloud": {
+    "availability_zone": "us-east-1c",
+    "instance_id": "i-4e123456",
+    "machine_type": "t2.medium",
+    "provider": "ec2",
+    "region": "us-east-1"
   }
 }
 -------------------------------------------------------------------------------
@@ -539,12 +537,10 @@ _Digital Ocean_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "instance_id": "1234567",
-      "provider": "digitalocean",
-      "region": "nyc2"
-    }
+  "cloud": {
+    "instance_id": "1234567",
+    "provider": "digitalocean",
+    "region": "nyc2"
   }
 }
 -------------------------------------------------------------------------------
@@ -554,14 +550,12 @@ _GCE_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "availability_zone": "projects/1234567890/zones/us-east1-b",
-      "instance_id": "1234556778987654321",
-      "machine_type": "projects/1234567890/machineTypes/f1-micro",
-      "project_id": "my-dev",
-      "provider": "gce"
-    }
+  "cloud": {
+    "availability_zone": "us-east1-b",
+    "instance_id": "1234556778987654321",
+    "machine_type": "f1-micro",
+    "project_id": "my-dev",
+    "provider": "gce"
   }
 }
 -------------------------------------------------------------------------------
@@ -571,13 +565,11 @@ _Tencent Cloud_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "availability_zone": "gz-azone2",
-      "instance_id": "ins-qcloudv5",
-      "provider": "qcloud",
-      "region": "china-south-gz"
-    }
+  "cloud": {
+    "availability_zone": "gz-azone2",
+    "instance_id": "ins-qcloudv5",
+    "provider": "qcloud",
+    "region": "china-south-gz"
   }
 }
 -------------------------------------------------------------------------------
@@ -590,13 +582,11 @@ ECS instance.
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "availability_zone": "cn-shenzhen",
-      "instance_id": "i-wz9g2hqiikg0aliyun2b",
-      "provider": "ecs",
-      "region": "cn-shenzhen-a"
-    }
+  "cloud": {
+    "availability_zone": "cn-shenzhen",
+    "instance_id": "i-wz9g2hqiikg0aliyun2b",
+    "provider": "ecs",
+    "region": "cn-shenzhen-a"
   }
 }
 -------------------------------------------------------------------------------
@@ -606,14 +596,12 @@ _Azure Virtual Machine_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "provider": "az",
-      "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e",
-      "instance_name": "test-az-vm",
-      "machine_type": "Standard_D3_v2",
-      "region": "eastus2"
-    }
+  "cloud": {
+    "provider": "az",
+    "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e",
+    "instance_name": "test-az-vm",
+    "machine_type": "Standard_D3_v2",
+    "region": "eastus2"
   }
 }
 -------------------------------------------------------------------------------
@@ -623,14 +611,12 @@ _Openstack Nova_
 [source,json]
 -------------------------------------------------------------------------------
 {
-  "meta": {
-    "cloud": {
-      "provider": "openstack",
-      "instance_name": "test-998d932195.mycloud.tld",
-      "availability_zone": "xxxx-az-c",
-      "instance_id": "i-00011a84",
-      "machine_type": "m2.large"
-    }
+  "cloud": {
+    "provider": "openstack",
+    "instance_name": "test-998d932195.mycloud.tld",
+    "availability_zone": "xxxx-az-c",
+    "instance_id": "i-00011a84",
+    "machine_type": "m2.large"
   }
 }
 -------------------------------------------------------------------------------

diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go
@@ -18,6 +18,8 @@
 package add_cloud_metadata
 
 import (
+	"path"
+
 	"github.com/elastic/beats/libbeat/common"
 	s "github.com/elastic/beats/libbeat/common/schema"
 	c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
@@ -30,6 +32,18 @@ func newGceMetadataFetcher(config *common.Config) (*metadataFetcher, error) {
 	gceSchema := func(m map[string]interface{}) common.MapStr {
 		out := common.MapStr{}
 
+		trimLeadingPath := func(key string) {
+			v, err := out.GetValue(key)
+			if err != nil {
+				return
+			}
+			p, ok := v.(string)
+			if !ok {
+				return
+			}
+			out.Put(key, path.Base(p))
+		}
+
 		if instance, ok := m["instance"].(map[string]interface{}); ok {
 			s.Schema{
 				"instance": s.Object{
@@ -41,6 +55,8 @@ func newGceMetadataFetcher(config *common.Config) (*metadataFetcher, error) {
 				},
 				"availability_zone": c.Str("zone"),
 			}.ApplyTo(out, instance)
+			trimLeadingPath("machine.type")
+			trimLeadingPath("availability_zone")
 		}
 
 		if project, ok := m["project"].(map[string]interface{}); ok {

diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go
@@ -158,9 +158,9 @@ func TestRetrieveGCEMetadata(t *testing.T) {
 				"name": "test-gce-dev",
 			},
 			"machine": common.MapStr{
-				"type": "projects/111111111111/machineTypes/f1-micro",
+				"type": "f1-micro",
 			},
-			"availability_zone": "projects/111111111111/zones/us-east1-b",
+			"availability_zone": "us-east1-b",
 			"project": common.MapStr{
 				"id": "test-dev",
 			},