Cherry-pick #21175 to 7.x: Some fixes in Cloud Foundry fields

CHANGELOG.next.asciidoc

@@ -29,6 +29,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Ensure dynamic template names are unique for the same field. {pull}18849[18849]
 - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 - Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
+- Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]
 
 *Auditbeat*
 

x-pack/libbeat/common/cloudfoundry/events.go

@@ -492,10 +492,7 @@ func envelopMap(evt Event) common.MapStr {
 func baseMap(evt Event) common.MapStr {
 	return common.MapStr{
 		"cloudfoundry": common.MapStr{
-			"type": evt.String(),
-			evt.String(): common.MapStr{
-				"timestamp": evt.Timestamp(),
-			},
+			"type":     evt.String(),
 			"envelope": envelopMap(evt),
 		},
 	}

x-pack/libbeat/common/cloudfoundry/events_test.go

@@ -76,9 +76,6 @@ func TestEventTypeHttpAccess(t *testing.T) {
 	assert.Equal(t, common.MapStr{
 		"cloudfoundry": common.MapStr{
 			"type": "access",
-			"access": common.MapStr{
-				"timestamp": time.Unix(0, 1587469726082),
-			},
 			"envelope": common.MapStr{
 				"origin":     "origin",
 				"deployment": "deployment",
@@ -148,7 +145,6 @@ func TestEventTypeLog(t *testing.T) {
 		"cloudfoundry": common.MapStr{
 			"type": "log",
 			"log": common.MapStr{
-				"timestamp": time.Unix(0, 1587469726082),
 				"source": common.MapStr{
 					"instance": evt.SourceID(),
 					"type":     evt.SourceType(),
@@ -200,10 +196,9 @@ func TestEventCounter(t *testing.T) {
 		"cloudfoundry": common.MapStr{
 			"type": "counter",
 			"counter": common.MapStr{
-				"timestamp": time.Unix(0, 1587469726082),
-				"name":      "name",
-				"delta":     uint64(10),
-				"total":     uint64(999),
+				"name":  "name",
+				"delta": uint64(10),
+				"total": uint64(999),
 			},
 			"envelope": common.MapStr{
 				"origin":     "origin",
@@ -246,10 +241,9 @@ func TestEventValueMetric(t *testing.T) {
 		"cloudfoundry": common.MapStr{
 			"type": "value",
 			"value": common.MapStr{
-				"timestamp": time.Unix(0, 1587469726082),
-				"name":      "name",
-				"value":     10.1,
-				"unit":      "unit",
+				"name":  "name",
+				"value": 10.1,
+				"unit":  "unit",
 			},
 			"envelope": common.MapStr{
 				"origin":     "origin",
@@ -304,7 +298,6 @@ func TestEventContainerMetric(t *testing.T) {
 		"cloudfoundry": common.MapStr{
 			"type": "container",
 			"container": common.MapStr{
-				"timestamp":          time.Unix(0, 1587469726082),
 				"instance_index":     int32(1),
 				"cpu.pct":            0.2,
 				"memory.bytes":       uint64(1024),
@@ -356,8 +349,7 @@ func TestEventError(t *testing.T) {
 		"cloudfoundry": common.MapStr{
 			"type": "error",
 			"error": common.MapStr{
-				"timestamp": time.Unix(0, 1587469726082),
-				"source":    "source",
+				"source": "source",
 			},
 			"envelope": common.MapStr{
 				"origin":     "origin",

x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml

@@ -11,29 +11,57 @@
         - name: app.id
           type: keyword
           description: >
-            Cloud Foundry application ID
+            Cloud Foundry application identifier.
 
         - name: app.name
           type: keyword
           description: >
-            Cloud Foundry application name
+            Cloud Foundry application name.
 
         - name: space.id
           type: keyword
           description: >
-            Cloud Foundry space name
+            Cloud Foundry space identifier.
 
         - name: space.name
           type: keyword
           description: >
-            Cloud Foundry space name
+            Cloud Foundry space name.
 
         - name: org.id
           type: keyword
           description: >
-            Cloud Foundry organization ID
+            Cloud Foundry organization identifier.
 
         - name: org.name
           type: keyword
           description: >
-            Cloud Foundry organization name
+            Cloud Foundry organization name.
+
+        - name: envelope
+          type: group
+          fields:
+            - name: deployment
+              type: keyword
+              description: >
+                Identifier of the Cloud Foundry deployment where this event was created.
+
+            - name: index
+              type: keyword
+              description: >
+                Envelope index.
+
+            - name: ip
+              type: ip
+              description: >
+                IP address of the node where the job that created this event is running.
+
+            - name: job
+              type: keyword
+              description: >
+                Job name.
+
+            - name: origin
+              type: keyword
+              description: >
+                Name of the deployment that created this event.

x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json

@@ -1,35 +1,27 @@
 {
-    "@timestamp": "2020-02-27T18:09:35.583Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
+        "app": {
+            "id": "c0874969-2811-4d80-837a-301fd485a7bf"
+        },
         "container": {
-            "instance_index": 0,
-            "cpu.pct": 0.18815706013687805,
-            "memory.bytes": 23320985,
-            "memory.quota.bytes": 134217728,
-            "disk.bytes": 186187776,
-            "timestamp": "2020-02-27T18:09:35.583Z",
-            "type": "container",
-            "disk.quota.bytes": 2122136037
+            "cpu.pct": 0.45456369228159,
+            "disk.bytes": 21970944,
+            "disk.quota.bytes": 1073741824,
+            "instance_index": 1,
+            "memory.bytes": 14101162,
+            "memory.quota.bytes": 67108864
         },
         "envelope": {
-            "origin": "rep",
-            "deployment": "cf",
-            "ip": "10.144.0.13",
-            "job": "diego-cell",
-            "index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d",
+            "ip": "192.168.16.31",
+            "job": "diego_cell",
+            "origin": "rep"
         },
-        "app": {
-            "id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a"
-        }
-    },
-    "event": {
-        "dataset": "cloudfoundry.container",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "container"
+        "type": "container"
     },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}

x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json

@@ -1,29 +1,21 @@
 {
-    "@timestamp": "2020-02-24T18:49:02.674Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
         "counter": {
-            "total": 0,
-            "name": "promhttp_metric_handler_errors_total",
             "delta": 0,
-            "timestamp": "2020-02-24T18:49:02.674Z",
-            "type": "counter"
+            "name": "sinks.dropped",
+            "total": 89547093
         },
         "envelope": {
-            "origin": "loggregator_forwarder_agent",
-            "deployment": "cf",
-            "ip": "10.144.0.13",
-            "job": "diego-cell",
-            "index": "1e58e943-a498-4339-9d5f-2885bf223db9"
-        }
-    },
-    "event": {
-        "dataset": "cloudfoundry.counter",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "counter"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "9adedbd0-3af3-47af-9707-99016050b123",
+            "ip": "192.168.16.54",
+            "job": "doppler",
+            "origin": "loggregator.doppler"
+        },
+        "type": "counter"
     },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}

x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json

@@ -1,29 +1,21 @@
 {
-    "@timestamp": "2020-02-27T18:20:48.379Z",
+    "@timestamp": "2017-10-12T08:05:34.853Z",
     "cloudfoundry": {
-        "value": {
-            "unit": "",
-            "value": 5.125735912782789e-05,
-            "timestamp": "2020-02-27T18:20:48.379Z",
-            "type": "value",
-            "name": "go_memstats_gc_cpu_fraction"
-        },
         "envelope": {
-            "origin": "loggregator_forwarder_agent",
-            "deployment": "cf",
-            "ip": "10.144.0.11",
-            "job": "scheduler",
-            "index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f"
+            "deployment": "cf-6b7aee31c8d07637ad78",
+            "index": "439bb519-3e68-4d08-a973-8f013f364d61",
+            "ip": "192.168.16.18",
+            "job": "cloud_controller",
+            "origin": "routing_api"
+        },
+        "type": "value",
+        "value": {
+            "name": "memoryStats.lastGCPauseTimeNS",
+            "unit": "count",
+            "value": 77499
         }
     },
-    "event": {
-        "dataset": "cloudfoundry.value",
-        "module": "cloudfoundry"
-    },
-    "metricset": {
-        "name": "value"
-    },
     "service": {
         "type": "cloudfoundry"
     }
-}
+}