Add cloudfoundry tags to events (elastic#21177)

jsoriano · Sep 22, 2020 · c9ea2cf · c9ea2cf
1 parent aebddba
commit c9ea2cf
Show file tree

Hide file tree

Showing 11 changed files with 78 additions and 22 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -433,6 +433,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added experimental dataset `fortinet/fortimanager`. {pull}20820[20820]
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
+- Add Cloud Foundry tags in related events. {pull}21177[21177]
 
 *Auditbeat*
 

diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go
@@ -494,10 +494,19 @@ func baseMap(evt Event) common.MapStr {
 		"cloudfoundry": common.MapStr{
 			"type":     evt.String(),
 			"envelope": envelopMap(evt),
+			"tags":     dedotedTags(evt.Tags()),
 		},
 	}
 }
 
+func dedotedTags(tags map[string]string) common.MapStr {
+	result := common.MapStr{}
+	for name, value := range tags {
+		result[common.DeDot(name)] = value
+	}
+	return result
+}
+
 func baseMapWithApp(evt EventWithAppID) common.MapStr {
 	base := baseMap(evt)
 	appID := evt.AppGuid()

diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go
@@ -86,6 +86,9 @@ func TestEventTypeHttpAccess(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"http": common.MapStr{
 			"response": common.MapStr{
@@ -160,6 +163,9 @@ func TestEventTypeLog(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "log message",
 		"stream":  "stdout",
@@ -207,6 +213,9 @@ func TestEventCounter(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -252,6 +261,9 @@ func TestEventValueMetric(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -315,6 +327,9 @@ func TestEventContainerMetric(t *testing.T) {
 			"app": common.MapStr{
 				"id": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 	}, evt.ToFields())
 }
@@ -358,6 +373,9 @@ func TestEventError(t *testing.T) {
 				"job":        "job",
 				"index":      "index",
 			},
+			"tags": common.MapStr{
+				"tag": "value",
+			},
 		},
 		"message": "message",
 		"code":    int32(100),

diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml
@@ -38,6 +38,12 @@
           description: >
             Cloud Foundry organization name.
 
+        - name: tags.*
+          type: object
+          object_type: keyword
+          description: >
+            Cloud Foundry tags.
+
         - name: envelope
           type: group
           fields:

diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json
@@ -2,23 +2,27 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "app": {
-            "id": "c0874969-2811-4d80-837a-301fd485a7bf"
+            "id": "3ce55e14-de73-49af-836d-adc93f3fee39"
         },
         "container": {
-            "cpu.pct": 0.45456369228159,
-            "disk.bytes": 21970944,
-            "disk.quota.bytes": 1073741824,
-            "instance_index": 1,
-            "memory.bytes": 14101162,
-            "memory.quota.bytes": 67108864
+            "cpu.pct": 0.19431789913648675,
+            "disk.bytes": 16678912,
+            "disk.quota.bytes": 33554432,
+            "instance_index": 0,
+            "memory.bytes": 8529920,
+            "memory.quota.bytes": 33554432
         },
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
-            "ip": "192.168.16.31",
+            "index": "c2bcf5d6-7ff9-4876-890f-6f8fc6c58668",
+            "ip": "192.168.16.51",
             "job": "diego_cell",
             "origin": "rep"
         },
+        "tags": {
+            "product": "Pivotal Application Service",
+            "source_id": "3ce55e14-de73-49af-836d-adc93f3fee39"
+        },
         "type": "container"
     },
     "service": {

diff --git a/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/container/container_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json
@@ -3,15 +3,21 @@
     "cloudfoundry": {
         "counter": {
             "delta": 0,
-            "name": "sinks.dropped",
-            "total": 89547093
+            "name": "dropped",
+            "total": 0
         },
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "9adedbd0-3af3-47af-9707-99016050b123",
-            "ip": "192.168.16.54",
-            "job": "doppler",
-            "origin": "loggregator.doppler"
+            "index": "995eb273-f871-4fea-a834-dbc0a4a72efc",
+            "ip": "192.168.16.37",
+            "job": "syslog_adapter",
+            "origin": "loggregator.metron"
+        },
+        "tags": {
+            "direction": "ingress",
+            "metric_version": "2.0",
+            "product": "Pivotal Application Service",
+            "source_id": "metron"
         },
         "type": "counter"
     },

diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/counter/counter_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }
diff --git a/x-pack/metricbeat/module/cloudfoundry/mtest/config.go b/x-pack/metricbeat/module/cloudfoundry/mtest/config.go
@@ -7,6 +7,7 @@ package mtest
 import (
 	"testing"
 
+	"github.com/elastic/beats/v7/libbeat/beat"
 	cftest "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry/test"
 )
 
@@ -19,3 +20,8 @@ func GetConfig(t *testing.T, metricset string) map[string]interface{} {
 
 	return config
 }
+
+// CleanFields removes fields that can contain data of real deployments
+func CleanFields(e beat.Event) {
+	e.Fields.Delete("cloudfoundry.tags.system_domain")
+}
diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json
@@ -3,16 +3,19 @@
     "cloudfoundry": {
         "envelope": {
             "deployment": "cf-6b7aee31c8d07637ad78",
-            "index": "439bb519-3e68-4d08-a973-8f013f364d61",
-            "ip": "192.168.16.18",
-            "job": "cloud_controller",
-            "origin": "routing_api"
+            "index": "dffd1799-d03a-405a-9309-3fcce23f536f",
+            "ip": "192.168.16.15",
+            "job": "diego_database",
+            "origin": "silk-controller"
+        },
+        "tags": {
+            "source_id": "silk-controller"
         },
         "type": "value",
         "value": {
-            "name": "memoryStats.lastGCPauseTimeNS",
-            "unit": "count",
-            "value": 77499
+            "name": "LeasesIndexRequestTime",
+            "unit": "ms",
+            "value": 0.681265
         }
     },
     "service": {

diff --git a/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go b/x-pack/metricbeat/module/cloudfoundry/value/value_integration_test.go
@@ -49,5 +49,6 @@ func TestData(t *testing.T) {
 	require.NotEmpty(t, events)
 
 	beatEvent := mbtest.StandardizeEvent(ms, events[0])
+	mtest.CleanFields(beatEvent)
 	mbtest.WriteEventToDataJSON(t, beatEvent, "")
 }