Updating Metricbeat stack modules to ECS

CHANGELOG.next.asciidoc

@@ -101,6 +101,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   - mongodb.metrics.replication.executor.network_interface
   - php_fpm.process.request_uri
   - php_fpm.process.script
+- Update a few elasticsearch.* fields to map to ECS. {pull}10350[10350]
+- Update a few logstash.* fields to map to ECS. {pull}10350[10350]
+- Update a few kibana.* fields to map to ECS. {pull}10350[10350]
 
 *Packetbeat*
 

dev-tools/ecs-migration.yml

@@ -1573,6 +1573,50 @@
   alias: true
   beat: metricbeat
 
+### Kibana
+
+- from: kibana.stats.uuid
+  to: service.id
+  alias: true
+  beat: metricbeat
+
+- from: kibana.stats.transport_address
+  to: service.address
+  alias: true
+  beat: metricbeat
+
+- from: kibana.stats.version
+  to: service.version
+  alias: true
+  beat: metricbeat
+
+- from: kibana.status.uuid
+  to: service.id
+  alias: true
+  beat: metricbeat
+
+- from: kibana.status.version.number
+  to: service.version
+  alias: true
+  beat: metricbeat
+
+### Logstash
+
+- from: logstash.node.host
+  to: service.hostname
+  alias: true
+  beat: metricbeat
+
+- from: logstash.node.version
+  to: service.version
+  alias: true
+  beat: metricbeat
+
+- from: logstash.node.jvm.pid
+  to: process.pid
+  alias: true
+  beat: metricbeat
+
 ### Zookeeper
 
 - from: zookeeper.mntr.version

metricbeat/_meta/fields.common.yml

@@ -22,9 +22,13 @@
 
     - name: service.address
       description: >
-        Connection address of the machine from which the metricset was collected. This
+        Address of the machine where the service is running. This
         field may not be present when the data was collected locally.
 
+    - name: service.hostname
+      description: >
+        Host name of the machine where the service is running.
+
     - name: type
       required: true
       example: metricsets

metricbeat/docs/fields.asciidoc

@@ -1990,7 +1990,15 @@ Process group id.
 *`service.address`*::
 +
 --
-Connection address of the machine from which the metricset was collected. This field may not be present when the data was collected locally.
+Address of the machine where the service is running. This field may not be present when the data was collected locally.
+
+
+--
+
+*`service.hostname`*::
++
+--
+Host name of the machine where the service is running.
 
 
 --
@@ -7485,9 +7493,9 @@ Target node id.
 *`elasticsearch.index.recovery.target.host`*::
 +
 --
-type: ip
+type: keyword
 
-Target node host address.
+Target node host address (could be IP address or hostname).
 
 
 --
@@ -7515,9 +7523,9 @@ Source node id.
 *`elasticsearch.index.recovery.source.host`*::
 +
 --
-type: ip
+type: keyword
 
-Source node host address.
+Source node host address (could be IP address or hostname).
 
 
 --
@@ -11028,7 +11036,9 @@ Kibana stats and run-time metrics.
 *`kibana.stats.uuid`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.id
 
 Kibana instance UUID
 
@@ -11068,7 +11078,9 @@ Kibana instance hostname
 *`kibana.stats.transport_address`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.address
 
 Kibana server's hostname and port
 
@@ -11078,7 +11090,9 @@ Kibana server's hostname and port
 *`kibana.stats.version`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.version
 
 Kibana version
 
@@ -11259,7 +11273,9 @@ Kibana instance name.
 *`kibana.status.uuid`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.id
 
 Kibana instance uuid.
 
@@ -11269,7 +11285,9 @@ Kibana instance uuid.
 *`kibana.status.version.number`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.version
 
 Kibana version number.
 
@@ -13309,7 +13327,9 @@ node
 *`logstash.node.host`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: host.hostname
 
 Host name
 
@@ -13319,7 +13339,9 @@ Host name
 *`logstash.node.version`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: service.version
 
 Logstash Version
 
@@ -13346,9 +13368,11 @@ Version
 *`logstash.node.jvm.pid`*::
 +
 --
-type: long
+type: alias
+
+alias to: process.pid
 
-Pid
+Process ID
 
 
 --

metricbeat/module/elasticsearch/index_recovery/_meta/fields.yml

@@ -26,9 +26,9 @@
       description: >
         Target node id.
     - name: target.host
-      type: ip
+      type: keyword
       description: >
-        Target node host address.
+        Target node host address (could be IP address or hostname).
     - name: target.name
       type: keyword
       description: >
@@ -39,9 +39,9 @@
       description: >
         Source node id.
     - name: source.host
-      type: ip
+      type: keyword
       description: >
-        Source node host address.
+        Source node host address (could be IP address or hostname).
     - name: source.name
       type: keyword
       description: >

metricbeat/module/kibana/stats/_meta/data.json

@@ -1,41 +1,46 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
+    "agent": {
         "hostname": "host.example.com",
         "name": "host.example.com"
     },
     "elasticsearch": {
         "cluster": {
-            "id": "njqU4EQaTROIDlWPeUMQyw"
+            "id": "hyK0oDhoThywCtJd2DV8Bg"
         }
     },
+    "event": {
+        "dataset": "kibana.stats",
+        "duration": 115000,
+        "module": "kibana"
+    },
     "kibana": {
         "stats": {
-            "concurrent_connections": 6,
+            "concurrent_connections": 1,
             "host": {
                 "name": "localhost"
             },
-            "index": "Shaunaks-MBP-2",
-            "name": "Shaunaks-MBP-2",
+            "index": "Shaunaks-MBP-2.attlocal.net",
+            "name": "Shaunaks-MBP-2.attlocal.net",
             "process": {
                 "event_loop_delay": {
-                    "ms": 0.23628300055861473
+                    "ms": 0.45250000059604645
                 },
                 "memory": {
                     "heap": {
                         "size_limit": {
-                            "bytes": 1501560832
+                            "bytes": 1526909922
                         },
                         "total": {
-                            "bytes": 215003136
+                            "bytes": 307593216
                         },
                         "used": {
-                            "bytes": 185343400
+                            "bytes": 226887112
                         }
                     }
                 },
                 "uptime": {
-                    "ms": 1343714
+                    "ms": 124098
                 }
             },
             "request": {
@@ -49,22 +54,20 @@
                 }
             },
             "snapshot": false,
-            "status": "green",
-            "transport_address": "localhost:5601",
-            "uuid": "5b2de169-2785-441b-ae8c-186a1936b17d",
-            "version": "7.0.0-alpha1"
+            "status": "green"
         }
     },
     "metricset": {
-        "host": "127.0.0.1:5601",
-        "module": "kibana",
-        "name": "stats",
-        "rtt": 115
+        "name": "stats"
     },
     "process": {
-        "pid": 20173
+        "pid": 93807
     },
     "service": {
-        "name": "kibana"
+        "address": "127.0.0.1:5601",
+        "id": "5b2de169-2785-441b-ae8c-186a1936b17d",
+        "name": "kibana",
+        "type": "kibana",
+        "version": "7.0.0"
     }
 }

metricbeat/module/kibana/stats/_meta/fields.yml

@@ -5,7 +5,9 @@
   release: ga
   fields:
     - name: uuid
-      type: keyword
+      type: alias
+      path: service.id
+      migration: true
       description: >
         Kibana instance UUID
     - name: name
@@ -21,13 +23,17 @@
       description: >
         Kibana instance hostname
     - name: transport_address
-      type: keyword
+      type: alias
+      path: service.address
+      migration: true
       description: >
         Kibana server's hostname and port
     - name: version
-      type: keyword
+      type: alias
       description: >
         Kibana version
+      path: service.version
+      migration: true
     - name: snapshot
       type: boolean
       description: >