Add cloudfoundry tags to events (#21177) (#21223)

(cherry picked from commit c9ea2cf)
elastic · Sep 23, 2020 · 1a547ef · 1a547ef
1 parent 95c4430
commit 1a547ef
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -506,6 +506,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
+- Add Cloud Foundry tags in related events. {pull}21177[21177]
 
 *Auditbeat*
 

diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go
@@ -494,10 +494,19 @@ func baseMap(evt Event) common.MapStr {
 		"cloudfoundry": common.MapStr{
 			"type":     evt.String(),
 			"envelope": envelopMap(evt),
+			"tags":     dedotedTags(evt.Tags()),
 		},
 	}
 }
 
+func dedotedTags(tags map[string]string) common.MapStr {
+	result := common.MapStr{}
+	for name, value := range tags {
+		result[common.DeDot(name)] = value
+	}
+	return result
+}
+
 func baseMapWithApp(evt EventWithAppID) common.MapStr {
 	base := baseMap(evt)
 	appID := evt.AppGuid()

diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go
@@ -86,6 +86,9 @@ func TestEventTypeHttpAccess(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"http": common.MapStr{
 			"response": common.MapStr{
@@ -160,6 +163,9 @@ func TestEventTypeLog(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "log message",
 		"stream":  "stdout",
@@ -207,6 +213,9 @@ func TestEventCounter(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -252,6 +261,9 @@ func TestEventValueMetric(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -315,6 +327,9 @@ func TestEventContainerMetric(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -358,6 +373,9 @@ func TestEventError(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "message",
 		"code":    int32(100),

diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml
@@ -38,6 +38,12 @@
           description: >
             Cloud Foundry organization name.
 
+        - name: tags.*
+          type: object
+          object_type: keyword
+          description: >
+            Cloud Foundry tags.
+
         - name: envelope
           type: group
           fields:
@@ -65,3 +71,4 @@
               type: keyword
               description: >
                 Name of the deployment that created this event.
+>>>>>>> origin/7.x
diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json
@@ -2,23 +2,27 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "app": {
-            "id": "c0874969-2811-4d80-837a-301fd485a7bf"
+            "id": "e6bb1d20-64e1-42ff-abda-b502eb8cb2dc"
         },
         "container": {
-            "cpu.pct": 0.45456369228159,
-            "disk.bytes": 21970944,
+            "cpu.pct": 0.2808890817947714,
+            "disk.bytes": 10100736,
             "disk.quota.bytes": 1073741824,
-            "instance_index": 1,
-            "memory.bytes": 14101162,
-            "memory.quota.bytes": 67108864
+            "instance_index": 3,
+            "memory.bytes": 14486732,
+            "memory.quota.bytes": 134217728
         },
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
-            "ip": "192.168.16.31",
+            "index": "cd6c6b2c-797b-49a9-b1a2-a3b52da80965",
+            "ip": "192.168.16.32",
             "job": "diego_cell",
             "origin": "rep"
         },
+        "tags": {
+            "product": "Pivotal Application Service",
+            "source_id": "e6bb1d20-64e1-42ff-abda-b502eb8cb2dc"
+        },
         "type": "container"
     },
     "service": {

diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json
@@ -2,17 +2,22 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "counter": {
-            "delta": 0,
-            "name": "sinks.dropped",
-            "total": 89547093
+            "delta": 201,
+            "name": "ingress",
+            "total": 512406486
         },
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "9adedbd0-3af3-47af-9707-99016050b123",
-            "ip": "192.168.16.54",
+            "index": "d0d52df7-be5e-44ec-b643-988dd9a85ca8",
+            "ip": "192.168.16.53",
             "job": "doppler",
             "origin": "loggregator.doppler"
         },
+        "tags": {
+            "metric_version": "2.0",
+            "product": "Pivotal Application Service",
+            "source_id": "doppler"
+        },
         "type": "counter"
     },
     "service": {

diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go
@@ -7,6 +7,7 @@ package mtest
 import (
 	"testing"
 
+	"github.com/elastic/beats/v7/libbeat/beat"
 	cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test"
 )
 
@@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} {
 
 	return config
 }
+
+// CleanFields removes fields that can contain data of real deployments
+func CleanFields(e beat.Event) {
+	e.Fields.Delete("cloudfoundry.tags.system_domain")
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json
@@ -3,16 +3,19 @@
     "cloudfoundry": {
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "439bb519-3e68-4d08-a973-8f013f364d61",
-            "ip": "192.168.16.18",
-            "job": "cloud_controller",
-            "origin": "routing_api"
+            "index": "6f2f5d74-9375-4d82-870f-c09e6a74cf3a",
+            "ip": "192.168.16.46",
+            "job": "diego_cell",
+            "origin": "netmon"
+        },
+        "tags": {
+            "source_id": "netmon"
         },
         "type": "value",
         "value": {
             "name": "memoryStats.lastGCPauseTimeNS",
             "unit": "count",
-            "value": 77499
+            "value": 366021
         }
     },
     "service": {

diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }