Rename thread id fields in filebeat module for mysql

Several ids in the filebeat module for mysql are named differently but they all refer to thread (or connection) ids. They are internal mysql ids that don't map to operating system processes or threads. Use `mysql.thread_id` for these ids. Also remove timestamp fields after converting them to dates.
elastic · Jan 18, 2019 · e8e82ec · e8e82ec
1 parent 4140305
commit e8e82ec
Show file tree

Hide file tree

Showing 14 changed files with 55 additions and 71 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -42,6 +42,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033]
 - Now save the 'first seen' timestamp in `event.created` (previously `read_timestamp`),
   instead of saving the parsed date. Now aligned with `event.created` semantics elsewhere. {pull}10139[10139]
+- Rename `mysql.error.thread_id` and `mysql.slowlog.id` to `mysql.thread_id`. {pull}10161[10161]
+- Remove `mysql.error.timestamp`  and `mysql.slowlog.timestamp`. {pull}10161[10161]
 
 *Heartbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -551,7 +551,7 @@
   alias: true
 
 - from: mysql.error.thread_id
-  to: process.thread.id
+  to: mysql.thread_id
   alias: true
 
 - from: mysql.slowlog.ip
@@ -566,6 +566,10 @@
   to: user.name
   alias: true
 
+- from: mysql.slowlog.id
+  to: mysql.thread_id
+  alias: true
+
 ## NGINX module
 
 - from: nginx.access.user_name

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6322,27 +6322,29 @@ Fields from the MySQL log files.
 
 
 
-[float]
-== error fields
-
-Contains fields from the MySQL error logs.
+*`mysql.thread_id`*::
++
+--
+type: long
 
+The connection or thread ID for the query.
 
 
-*`mysql.error.timestamp`*::
-+
 --
-The timestamp from the log line.
+
+[float]
+== error fields
+
+Contains fields from the MySQL error logs.
 
 
---
 
 *`mysql.error.thread_id`*::
 +
 --
 type: alias
 
-alias to: process.thread.id
+alias to: mysql.thread_id
 
 --
 
@@ -6419,16 +6421,6 @@ type: long
 The number of rows modified by the query.
 
 
---
-
-*`mysql.slowlog.timestamp`*::
-+
---
-type: long
-
-The unix timestamp taken from the `SET timestamp` query.
-
-
 --
 
 *`mysql.slowlog.query`*::
@@ -6442,10 +6434,9 @@ The slow query.
 *`mysql.slowlog.id`*::
 +
 --
-type: long
-
-The connection or thread ID for the query.
+type: alias
 
+alias to: mysql.thread_id
 
 --
 

diff --git a/filebeat/module/mysql/_meta/fields.yml b/filebeat/module/mysql/_meta/fields.yml
@@ -9,3 +9,7 @@
       description: >
         Fields from the MySQL log files.
       fields:
+        - name: thread_id
+          type: long
+          description: >
+            The connection or thread ID for the query.
diff --git a/filebeat/module/mysql/error/_meta/fields.yml b/filebeat/module/mysql/error/_meta/fields.yml
@@ -3,13 +3,9 @@
   description: >
     Contains fields from the MySQL error logs.
   fields:
-    - name: timestamp
-      description: >
-        The timestamp from the log line.
-
     - name: thread_id
       type: alias
-      path: process.thread.id
+      path: mysql.thread_id
       migration: true
     - name: level
       type: alias

diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json
@@ -5,7 +5,7 @@
         "field": "message",
         "patterns": [
           "%{LOCALDATETIME:mysql.error.timestamp} (\\[%{DATA:log.level}\\] )?%{GREEDYDATA:message}",
-          "%{DATA:mysql.error.timestamp} %{NUMBER:process.thread.id:long} \\[%{DATA:log.level}\\] %{GREEDYDATA:message}",
+          "%{DATA:mysql.error.timestamp} %{NUMBER:mysql.thread_id:long} \\[%{DATA:log.level}\\] %{GREEDYDATA:message}",
           "%{GREEDYDATA:message}"
         ],
         "ignore_missing": true,
@@ -28,6 +28,11 @@
         ],
         "ignore_failure": true
       }
+    }, {
+      "remove":{
+        "field": "mysql.error.timestamp",
+        "ignore_missing": true
+      }
     }],
   "on_failure" : [{
     "set" : {

diff --git a/filebeat/module/mysql/error/test/error.log-expected.json b/filebeat/module/mysql/error/test/error.log-expected.json
@@ -8,7 +8,6 @@
         "input.type": "log",
         "log.offset": 0,
         "message": "mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql",
-        "mysql.error.timestamp": "161209 13:08:33",
         "service.type": "mysql"
     },
     {
@@ -21,8 +20,7 @@
         "log.level": "Warning",
         "log.offset": 92,
         "message": "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.335060Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -35,8 +33,7 @@
         "log.level": "Warning",
         "log.offset": 282,
         "message": "Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.335892Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -49,8 +46,7 @@
         "log.level": "Note",
         "log.offset": 478,
         "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 61571 ...",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.336610Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -63,8 +59,7 @@
         "log.level": "Warning",
         "log.offset": 603,
         "message": "Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.345527Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -77,8 +72,7 @@
         "log.level": "Note",
         "log.offset": 742,
         "message": "InnoDB: Mutexes and rw_locks use GCC atomic builtins",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.351596Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -91,8 +85,7 @@
         "log.level": "Note",
         "log.offset": 832,
         "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections.",
-        "mysql.error.timestamp": "2016-12-09T12:08:33.784722Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -115,8 +108,7 @@
         "log.level": "Note",
         "log.offset": 1002,
         "message": "InnoDB: page_cleaner: 1000ms intended loop took 772568ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.)",
-        "mysql.error.timestamp": "2016-12-09T22:21:02.443689Z",
-        "process.thread.id": 0,
+        "mysql.thread_id": 0,
         "service.type": "mysql"
     },
     {
@@ -129,7 +121,6 @@
         "log.level": "Warning",
         "log.offset": 1176,
         "message": "Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead.",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     },
     {
@@ -142,7 +133,6 @@
         "log.level": "Note",
         "log.offset": 1367,
         "message": "Plugin 'FEDERATED' is disabled.",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     },
     {
@@ -154,7 +144,6 @@
         "input.type": "log",
         "log.offset": 1422,
         "message": "InnoDB: The InnoDB memory heap is disabled",
-        "mysql.error.timestamp": "161209 14:18:50",
         "service.type": "mysql"
     }
 ]
diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go
diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml
@@ -24,17 +24,13 @@
       type: long
       description: >
         The number of rows modified by the query.
-    - name: timestamp
-      type: long
-      description: >
-        The unix timestamp taken from the `SET timestamp` query.
     - name: query
       description: >
         The slow query.
     - name: id
-      type: long
-      description: >
-        The connection or thread ID for the query.
+      type: alias
+      path: mysql.thread_id
+      migration: true
     - name: schema
       type: keyword
       description: >

diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.slowlog.id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
+        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\](\\s*Id:\\s* %{NUMBER:mysql.thread_id:long})?\n(# Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE}Schema: %{WORD:mysql.slowlog.schema}?%{METRICSPACE}QC_hit: %{WORD:mysql.slowlog.query_cache_hit}\n)?%{METRICSPACE}Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE}Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE}Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE}Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE}(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
         ],
       "pattern_definitions" : {
         "GREEDYMULTILINE": "(.|\n)*",
@@ -51,6 +51,11 @@
       "formats": ["UNIX"],
       "ignore_failure": true
     }
+  }, {
+    "remove":{
+      "field": "mysql.slowlog.timestamp",
+      "ignore_missing": true
+    }
   }, {
     "gsub": {
       "field": "mysql.slowlog.query",

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json
@@ -12,14 +12,13 @@
         ],
         "log.offset": 24,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "5",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2);",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 2.000652,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
-        "mysql.slowlog.timestamp": 1528898676,
+        "mysql.thread_id": "5",
         "service.type": "mysql",
         "source.domain": "localhost",
         "source.ip": "121.0.0.1",

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json
@@ -12,15 +12,14 @@
         ],
         "log.offset": 437,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "8",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2)\nAS foo;",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 2.000227,
         "mysql.slowlog.rows_affected": 0,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
-        "mysql.slowlog.timestamp": 1547741043,
+        "mysql.thread_id": "8",
         "service.type": "mysql",
         "source.domain": "localhost",
         "user.name": "root"
@@ -42,7 +41,6 @@
         "mysql.slowlog.filesort_on_disk": false,
         "mysql.slowlog.full_join": false,
         "mysql.slowlog.full_scan": true,
-        "mysql.slowlog.id": "25844",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.merge_passes": "0",
         "mysql.slowlog.priority_queue": false,
@@ -53,9 +51,9 @@
         "mysql.slowlog.rows_examined": 53022772,
         "mysql.slowlog.rows_sent": 0,
         "mysql.slowlog.schema": "blah",
-        "mysql.slowlog.timestamp": 1547741058,
         "mysql.slowlog.tmp_table": true,
         "mysql.slowlog.tmp_table_on_disk": false,
+        "mysql.thread_id": "25844",
         "service.type": "mysql",
         "source.ip": "192.168.0.10",
         "user.name": "root"

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json
@@ -12,15 +12,14 @@
         ],
         "log.offset": 24,
         "mysql.slowlog.current_user": "root",
-        "mysql.slowlog.id": "2",
         "mysql.slowlog.lock_time.sec": 0.000337,
         "mysql.slowlog.query": "select count(*) from customer, orders, nation\n  where c_custkey=o_custkey\n    and c_nationkey=n_nationkey\n    and n_name='GERMANY';",
         "mysql.slowlog.query_cache_hit": false,
         "mysql.slowlog.query_time.sec": 5.524103,
         "mysql.slowlog.rows_examined": 65633,
         "mysql.slowlog.rows_sent": 1,
         "mysql.slowlog.schema": "dbt3sf1",
-        "mysql.slowlog.timestamp": 1384261412,
+        "mysql.thread_id": "2",
         "service.type": "mysql",
         "source.domain": "localhost",
         "user.name": "root"