Add agent.{id,ephemeral_id} to all beat events (#9404)

* Add agent.{id,ephemeral_id} to all beat events This adds the ECS fields `agent.id` and `agent.ephemeral_id` to all beats. * Bring back whitespace * Use CHANGELOG.next * Update fb tests
elastic · Jan 3, 2019 · 700e982 · 700e982
1 parent b9a20f7
commit 700e982
Show file tree

Hide file tree

Showing 26 changed files with 122 additions and 117 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - Update field definitions for `http` to ECS Beta 2 {pull}9645[9645]
+- Add `agent.id` and `agent.ephemeral_id` fields to all beats. {pull}9404[9404]
 
 *Auditbeat*
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-09-20T15:42:59.000Z",
+        "@timestamp": "2019-09-20T15:42:59.000Z",
         "destination.ip": "1.2.3.4",
         "destination.port": 5000,
         "ecs.version": "1.0.0-beta2",

diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json
@@ -1,83 +1,83 @@
 [
     {
-        "@timestamp": "2018-08-07T08:27:47.000Z", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "slowlog", 
-        "event.module": "mysql", 
-        "input.type": "log", 
+        "@timestamp": "2018-08-07T08:27:47.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "slowlog",
+        "event.module": "mysql",
+        "input.type": "log",
         "log.flags": [
             "multiline"
-        ], 
-        "log.offset": 41, 
-        "mysql.slowlog.id": "7234", 
-        "mysql.slowlog.ip": "218.76.8.37", 
-        "mysql.slowlog.lock_time.sec": "0.000000", 
-        "mysql.slowlog.query": "select sleep(15);", 
-        "mysql.slowlog.query_time.sec": "15.000223", 
-        "mysql.slowlog.rows_examined": "0", 
-        "mysql.slowlog.rows_sent": "1", 
-        "mysql.slowlog.timestamp": "1533630467", 
+        ],
+        "log.offset": 41,
+        "mysql.slowlog.id": "7234",
+        "mysql.slowlog.ip": "218.76.8.37",
+        "mysql.slowlog.lock_time.sec": "0.000000",
+        "mysql.slowlog.query": "select sleep(15);",
+        "mysql.slowlog.query_time.sec": "15.000223",
+        "mysql.slowlog.rows_examined": "0",
+        "mysql.slowlog.rows_sent": "1",
+        "mysql.slowlog.timestamp": "1533630467",
         "mysql.slowlog.user": "root"
-    }, 
+    },
     {
-        "@timestamp": "2018-08-07T08:27:47.000Z", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "slowlog", 
-        "event.module": "mysql", 
-        "input.type": "log", 
+        "@timestamp": "2018-08-07T08:27:47.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "slowlog",
+        "event.module": "mysql",
+        "input.type": "log",
         "log.flags": [
             "multiline"
-        ], 
-        "log.offset": 254, 
-        "mysql.slowlog.host": "localhost", 
-        "mysql.slowlog.lock_time.sec": "0.000061", 
-        "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", 
-        "mysql.slowlog.query_time.sec": "0.000153", 
-        "mysql.slowlog.rows_examined": "5", 
-        "mysql.slowlog.rows_sent": "1", 
-        "mysql.slowlog.timestamp": "1533630467", 
+        ],
+        "log.offset": 254,
+        "mysql.slowlog.host": "localhost",
+        "mysql.slowlog.lock_time.sec": "0.000061",
+        "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';",
+        "mysql.slowlog.query_time.sec": "0.000153",
+        "mysql.slowlog.rows_examined": "5",
+        "mysql.slowlog.rows_sent": "1",
+        "mysql.slowlog.timestamp": "1533630467",
         "mysql.slowlog.user": "debian-sys-maint"
-    }, 
+    },
     {
-        "@timestamp": "2018-08-07T08:27:47.000Z", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "slowlog", 
-        "event.module": "mysql", 
-        "input.type": "log", 
+        "@timestamp": "2018-08-07T08:27:47.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "slowlog",
+        "event.module": "mysql",
+        "input.type": "log",
         "log.flags": [
             "multiline"
-        ], 
-        "log.offset": 526, 
-        "mysql.slowlog.host": "apphost", 
-        "mysql.slowlog.id": "10997316", 
-        "mysql.slowlog.ip": "1.1.1.1", 
-        "mysql.slowlog.lock_time.sec": "0.000212", 
-        "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n                    WHERE cus.cus_guid = mcu.cus_guid\n                        AND cus.pro_code = 'CYB'\n                        AND cus.cus_offline = 0\n                        AND mca.cus_guid = cus.cus_guid\n                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n                        AND mcu.mcu_crawlelements IS NOT NULL\n                    GROUP BY mcu.mcu_guid\n                    ORDER BY mcu.mcu_order ASC\n                    LIMIT 1000;", 
-        "mysql.slowlog.query_time.sec": "4.071491", 
-        "mysql.slowlog.rows_examined": "1489615", 
-        "mysql.slowlog.rows_sent": "1000", 
-        "mysql.slowlog.timestamp": "1533630467", 
+        ],
+        "log.offset": 526,
+        "mysql.slowlog.host": "apphost",
+        "mysql.slowlog.id": "10997316",
+        "mysql.slowlog.ip": "1.1.1.1",
+        "mysql.slowlog.lock_time.sec": "0.000212",
+        "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n                    WHERE cus.cus_guid = mcu.cus_guid\n                        AND cus.pro_code = 'CYB'\n                        AND cus.cus_offline = 0\n                        AND mca.cus_guid = cus.cus_guid\n                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n                        AND mcu.mcu_crawlelements IS NOT NULL\n                    GROUP BY mcu.mcu_guid\n                    ORDER BY mcu.mcu_order ASC\n                    LIMIT 1000;",
+        "mysql.slowlog.query_time.sec": "4.071491",
+        "mysql.slowlog.rows_examined": "1489615",
+        "mysql.slowlog.rows_sent": "1000",
+        "mysql.slowlog.timestamp": "1533630467",
         "mysql.slowlog.user": "apphost"
-    }, 
+    },
     {
-        "@timestamp": "2018-08-07T08:27:47.000Z", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "slowlog", 
-        "event.module": "mysql", 
-        "input.type": "log", 
+        "@timestamp": "2018-08-07T08:27:47.000Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "slowlog",
+        "event.module": "mysql",
+        "input.type": "log",
         "log.flags": [
             "multiline"
-        ], 
-        "log.offset": 1438, 
-        "mysql.slowlog.host": "apphost", 
-        "mysql.slowlog.id": "10999834", 
-        "mysql.slowlog.ip": "1.1.1.1", 
-        "mysql.slowlog.lock_time.sec": "0.000036", 
-        "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');", 
-        "mysql.slowlog.query_time.sec": "10.346539", 
-        "mysql.slowlog.rows_examined": "4751313", 
-        "mysql.slowlog.rows_sent": "0", 
-        "mysql.slowlog.timestamp": "1533630467", 
+        ],
+        "log.offset": 1438,
+        "mysql.slowlog.host": "apphost",
+        "mysql.slowlog.id": "10999834",
+        "mysql.slowlog.ip": "1.1.1.1",
+        "mysql.slowlog.lock_time.sec": "0.000036",
+        "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');",
+        "mysql.slowlog.query_time.sec": "10.346539",
+        "mysql.slowlog.rows_examined": "4751313",
+        "mysql.slowlog.rows_sent": "0",
+        "mysql.slowlog.timestamp": "1533630467",
         "mysql.slowlog.user": "apphost"
     }
 ]
diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-05-30T12:23:52.442Z",
+        "@timestamp": "2019-05-30T12:23:52.442Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "log",
         "event.module": "redis",
@@ -12,7 +12,7 @@
         "redis.log.role": "master"
     },
     {
-        "@timestamp": "2018-05-30T10:05:20.000Z",
+        "@timestamp": "2019-05-30T10:05:20.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "log",
         "event.module": "redis",
@@ -22,7 +22,7 @@
         "message": "0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects."
     },
     {
-        "@timestamp": "2018-05-31T04:32:08.000Z",
+        "@timestamp": "2019-05-31T04:32:08.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "log",
         "event.module": "redis",

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-02-21T21:54:44.000Z",
+        "@timestamp": "2019-02-21T21:54:44.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.action": "Accepted",
         "event.dataset": "auth",
@@ -16,7 +16,7 @@
         "user.name": "vagrant"
     },
     {
-        "@timestamp": "2018-02-23T00:13:35.000Z",
+        "@timestamp": "2019-02-23T00:13:35.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.action": "Accepted",
         "event.dataset": "auth",
@@ -31,7 +31,7 @@
         "user.name": "vagrant"
     },
     {
-        "@timestamp": "2018-02-21T21:56:12.000Z",
+        "@timestamp": "2019-02-21T21:56:12.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.action": "Invalid",
         "event.dataset": "auth",
@@ -44,7 +44,7 @@
         "user.name": "test"
     },
     {
-        "@timestamp": "2018-02-20T08:35:22.000Z",
+        "@timestamp": "2019-02-20T08:35:22.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.action": "Failed",
         "event.dataset": "auth",
@@ -65,7 +65,7 @@
         "user.name": "root"
     },
     {
-        "@timestamp": "2018-02-21T23:35:33.000Z",
+        "@timestamp": "2019-02-21T23:35:33.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",
@@ -79,7 +79,7 @@
         "user.name": "vagrant"
     },
     {
-        "@timestamp": "2018-02-19T15:30:04.000Z",
+        "@timestamp": "2019-02-19T15:30:04.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",
@@ -95,7 +95,7 @@
         "system.auth.ssh.dropped_ip": "123.57.245.163"
     },
     {
-        "@timestamp": "2018-02-23T00:08:48.000Z",
+        "@timestamp": "2019-02-23T00:08:48.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",
@@ -109,7 +109,7 @@
         "user.name": "vagrant"
     },
     {
-        "@timestamp": "2018-02-24T00:13:02.000Z",
+        "@timestamp": "2019-02-24T00:13:02.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",
@@ -124,7 +124,7 @@
         "user.name": "tsg"
     },
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z",
+        "@timestamp": "2019-02-22T11:47:05.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",
@@ -136,7 +136,7 @@
         "process.pid": 6991
     },
     {
-        "@timestamp": "2018-02-22T11:47:05.000Z",
+        "@timestamp": "2019-02-22T11:47:05.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "auth",
         "event.module": "system",

diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z",
+        "@timestamp": "2019-12-13T11:35:28.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "syslog",
         "event.module": "system",
@@ -15,7 +15,7 @@
         "process.pid": 21412
     },
     {
-        "@timestamp": "2018-12-13T11:35:28.000Z",
+        "@timestamp": "2019-12-13T11:35:28.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "syslog",
         "event.module": "system",
@@ -27,7 +27,7 @@
         "process.pid": 21412
     },
     {
-        "@timestamp": "2018-04-04T03:39:57.000Z",
+        "@timestamp": "2019-04-04T03:39:57.000Z",
         "ecs.version": "1.0.0-beta2",
         "event.dataset": "syslog",
         "event.module": "system",

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -195,7 +195,7 @@ def _test_expected_events(self, test_file, objects):
 
 def clean_keys(obj):
     # These keys are host dependent
-    host_keys = ["host.name", "agent.hostname", "agent.type"]
+    host_keys = ["host.name", "agent.hostname", "agent.type", "agent.ephemeral_id", "agent.id"]
     # The create timestamps area always new
     time_keys = ["read_timestamp", "event.created"]
     # source path and beat.version can be different for each run

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/_meta/fields.ecs.yml b/libbeat/_meta/fields.ecs.yml
@@ -73,12 +73,12 @@
           type: keyword
           description: >
             Name of the agent.
-    
+
             This is a name that can be given to an agent. This can be helpful if
             for example two Filebeat instances are running on the same host
             but a human readable separation is needed on which Filebeat instance
             data is coming from.
-    
+
             If no name is given, the name is often left empty.
           example: foo
 

diff --git a/libbeat/beat/info.go b/libbeat/beat/info.go
@@ -26,7 +26,8 @@ type Info struct {
 	Version     string    // The beat version. Defaults to the libbeat version when an implementation does not set a version
 	Name        string    // configured beat name
 	Hostname    string    // hostname
-	UUID        uuid.UUID // ID assigned to beat instance
+	ID          uuid.UUID // ID assigned to beat machine
+	EphemeralID uuid.UUID // ID assigned to beat process invocation (PID)
 
 	// Monitoring-related fields
 	Monitoring struct {