diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0d6c20f254a0..337a97aeb66a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -29,6 +29,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Ensure dynamic template names are unique for the same field. {pull}18849[18849] - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898] - Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293] +- Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175] *Auditbeat* diff --git a/x-pack/libbeat/common/cloudfoundry/events.go b/x-pack/libbeat/common/cloudfoundry/events.go index f134585ac656..3006c472ccd6 100644 --- a/x-pack/libbeat/common/cloudfoundry/events.go +++ b/x-pack/libbeat/common/cloudfoundry/events.go @@ -492,10 +492,7 @@ func envelopMap(evt Event) common.MapStr { func baseMap(evt Event) common.MapStr { return common.MapStr{ "cloudfoundry": common.MapStr{ - "type": evt.String(), - evt.String(): common.MapStr{ - "timestamp": evt.Timestamp(), - }, + "type": evt.String(), "envelope": envelopMap(evt), }, } diff --git a/x-pack/libbeat/common/cloudfoundry/events_test.go b/x-pack/libbeat/common/cloudfoundry/events_test.go index e4fe6f39caf4..911455899693 100644 --- a/x-pack/libbeat/common/cloudfoundry/events_test.go +++ b/x-pack/libbeat/common/cloudfoundry/events_test.go @@ -76,9 +76,6 @@ func TestEventTypeHttpAccess(t *testing.T) { assert.Equal(t, common.MapStr{ "cloudfoundry": common.MapStr{ "type": "access", - "access": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), - }, "envelope": common.MapStr{ "origin": "origin", "deployment": "deployment", @@ -148,7 +145,6 @@ func TestEventTypeLog(t *testing.T) { "cloudfoundry": common.MapStr{ "type": "log", "log": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), "source": common.MapStr{ "instance": evt.SourceID(), "type": evt.SourceType(), @@ -200,10 +196,9 @@ func TestEventCounter(t *testing.T) { "cloudfoundry": common.MapStr{ "type": "counter", "counter": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), - "name": "name", - "delta": uint64(10), - "total": uint64(999), + "name": "name", + "delta": uint64(10), + "total": uint64(999), }, "envelope": common.MapStr{ "origin": "origin", @@ -246,10 +241,9 @@ func TestEventValueMetric(t *testing.T) { "cloudfoundry": common.MapStr{ "type": "value", "value": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), - "name": "name", - "value": 10.1, - "unit": "unit", + "name": "name", + "value": 10.1, + "unit": "unit", }, "envelope": common.MapStr{ "origin": "origin", @@ -304,7 +298,6 @@ func TestEventContainerMetric(t *testing.T) { "cloudfoundry": common.MapStr{ "type": "container", "container": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), "instance_index": int32(1), "cpu.pct": 0.2, "memory.bytes": uint64(1024), @@ -356,8 +349,7 @@ func TestEventError(t *testing.T) { "cloudfoundry": common.MapStr{ "type": "error", "error": common.MapStr{ - "timestamp": time.Unix(0, 1587469726082), - "source": "source", + "source": "source", }, "envelope": common.MapStr{ "origin": "origin", diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml index bd6e158bb368..0ac4a813d502 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/_meta/fields.yml @@ -11,29 +11,57 @@ - name: app.id type: keyword description: > - Cloud Foundry application ID + Cloud Foundry application identifier. - name: app.name type: keyword description: > - Cloud Foundry application name + Cloud Foundry application name. - name: space.id type: keyword description: > - Cloud Foundry space name + Cloud Foundry space identifier. - name: space.name type: keyword description: > - Cloud Foundry space name + Cloud Foundry space name. - name: org.id type: keyword description: > - Cloud Foundry organization ID + Cloud Foundry organization identifier. - name: org.name type: keyword description: > - Cloud Foundry organization name + Cloud Foundry organization name. + + - name: envelope + type: group + fields: + - name: deployment + type: keyword + description: > + Identifier of the Cloud Foundry deployment where this event was created. + + - name: index + type: keyword + description: > + Envelope index. + + - name: ip + type: ip + description: > + IP address of the node where the job that created this event is running. + + - name: job + type: keyword + description: > + Job name. + + - name: origin + type: keyword + description: > + Name of the deployment that created this event. diff --git a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json index 3d1f51694e40..389b0bc54f95 100644 --- a/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/container/_meta/data.json @@ -1,35 +1,27 @@ { - "@timestamp": "2020-02-27T18:09:35.583Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { + "app": { + "id": "c0874969-2811-4d80-837a-301fd485a7bf" + }, "container": { - "instance_index": 0, - "cpu.pct": 0.18815706013687805, - "memory.bytes": 23320985, - "memory.quota.bytes": 134217728, - "disk.bytes": 186187776, - "timestamp": "2020-02-27T18:09:35.583Z", - "type": "container", - "disk.quota.bytes": 2122136037 + "cpu.pct": 0.45456369228159, + "disk.bytes": 21970944, + "disk.quota.bytes": 1073741824, + "instance_index": 1, + "memory.bytes": 14101162, + "memory.quota.bytes": 67108864 }, "envelope": { - "origin": "rep", - "deployment": "cf", - "ip": "10.144.0.13", - "job": "diego-cell", - "index": "3a0034ac-59ef-43e2-8b06-b14b32fe62f0" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "8895cf68-2e4f-4c50-9842-e73febd65c2d", + "ip": "192.168.16.31", + "job": "diego_cell", + "origin": "rep" }, - "app": { - "id": "8cfc6857-3576-45a9-b1cd-890b28aa9c4a" - } - }, - "event": { - "dataset": "cloudfoundry.container", - "module": "cloudfoundry" - }, - "metricset": { - "name": "container" + "type": "container" }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json index d6f155213af8..b9104346eac1 100644 --- a/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/counter/_meta/data.json @@ -1,29 +1,21 @@ { - "@timestamp": "2020-02-24T18:49:02.674Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { "counter": { - "total": 0, - "name": "promhttp_metric_handler_errors_total", "delta": 0, - "timestamp": "2020-02-24T18:49:02.674Z", - "type": "counter" + "name": "sinks.dropped", + "total": 89547093 }, "envelope": { - "origin": "loggregator_forwarder_agent", - "deployment": "cf", - "ip": "10.144.0.13", - "job": "diego-cell", - "index": "1e58e943-a498-4339-9d5f-2885bf223db9" - } - }, - "event": { - "dataset": "cloudfoundry.counter", - "module": "cloudfoundry" - }, - "metricset": { - "name": "counter" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "9adedbd0-3af3-47af-9707-99016050b123", + "ip": "192.168.16.54", + "job": "doppler", + "origin": "loggregator.doppler" + }, + "type": "counter" }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json index bf70762fd387..feeb2ce72ba8 100644 --- a/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json +++ b/x-pack/metricbeat/module/cloudfoundry/value/_meta/data.json @@ -1,29 +1,21 @@ { - "@timestamp": "2020-02-27T18:20:48.379Z", + "@timestamp": "2017-10-12T08:05:34.853Z", "cloudfoundry": { - "value": { - "unit": "", - "value": 5.125735912782789e-05, - "timestamp": "2020-02-27T18:20:48.379Z", - "type": "value", - "name": "go_memstats_gc_cpu_fraction" - }, "envelope": { - "origin": "loggregator_forwarder_agent", - "deployment": "cf", - "ip": "10.144.0.11", - "job": "scheduler", - "index": "d6b7bb59-ac57-4ed5-a0a7-0b23e19d9f8f" + "deployment": "cf-6b7aee31c8d07637ad78", + "index": "439bb519-3e68-4d08-a973-8f013f364d61", + "ip": "192.168.16.18", + "job": "cloud_controller", + "origin": "routing_api" + }, + "type": "value", + "value": { + "name": "memoryStats.lastGCPauseTimeNS", + "unit": "count", + "value": 77499 } }, - "event": { - "dataset": "cloudfoundry.value", - "module": "cloudfoundry" - }, - "metricset": { - "name": "value" - }, "service": { "type": "cloudfoundry" } -} +} \ No newline at end of file