From 383ccceb88e93139d5c18413e6597d739100c510 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 11 Jan 2019 14:08:24 -0500 Subject: [PATCH] Convert Filebeat mysql.* to ECS (#10008) - Convert many fields under `mysql.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field: - mysql.error.message => message - mysql.error.level => log.level - mysql.error.thread_id => process.thread.id - mysql.slowlog.ip => source.ip - mysql.slowlog.host => source.domain - mysql.slowlog.user => user.name - Add expected test file to test changes to error logs - Coerce int fields: process.thread.id, mysql.slowlog.id, mysql.slowlog.rows_sent, mysql.slowlog.rows_examined and mysql.slowlog.timestamp - Coerce float fields: mysql.slowlog.query_time.sec, mysql.slowlog.lock_time.sec - Set event.created with the `@timestamp` from Filebeat - Change username in test file from 'apphost' to 'appuser', to make it easier to distinguish them in the expected file - Populate event.duration based on mysql.slowlog.query_time.sec - Get rid of a workaround for Ingest Node that was fixed in 5.0 --- CHANGELOG.next.asciidoc | 1 + dev-tools/ecs-migration.yml | 26 +++ filebeat/docs/fields.asciidoc | 65 ++++---- filebeat/module/mysql/error/_meta/fields.yml | 20 +-- .../module/mysql/error/ingest/pipeline.json | 30 ++-- filebeat/module/mysql/error/test/error.log | 12 ++ .../mysql/error/test/error.log-expected.json | 148 ++++++++++++++++++ filebeat/module/mysql/fields.go | 2 +- .../module/mysql/slowlog/_meta/fields.yml | 22 +-- .../module/mysql/slowlog/ingest/pipeline.json | 8 +- .../mysql/slowlog/test/mysql-5.7.22.log | 4 +- .../test/mysql-5.7.22.log-expected.json | 70 +++++---- 12 files changed, 299 insertions(+), 109 deletions(-) create mode 100644 filebeat/module/mysql/error/test/error.log create mode 100644 filebeat/module/mysql/error/test/error.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cc59e824b06..643b688340c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -29,6 +29,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810] - Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935] +- Rename a few `mysql.*` fields to map to ECS. {pull}10008[10008] - Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007] *Heartbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index da779dfb87f..7b503928c41 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -466,6 +466,32 @@ to: message alias: true +## MySQL module + +- from: mysql.error.message + to: message + alias: true + +- from: mysql.error.level + to: log.level + alias: true + +- from: mysql.error.thread_id + to: process.thread.id + alias: true + +- from: mysql.slowlog.ip + to: source.ip + alias: true + +- from: mysql.slowlog.host + to: source.domain + alias: true + +- from: mysql.slowlog.user + to: user.name + alias: true + ## NGINX module - from: nginx.access.user_name diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4bd74761b4f..8dacbd02730 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -6339,29 +6339,27 @@ The timestamp from the log line. *`mysql.error.thread_id`*:: + -- -type: long - -As of MySQL 5.7.2, this is the thread id. For MySQL versions prior to 5.7.2, this field contains the process id. +type: alias +alias to: process.thread.id -- *`mysql.error.level`*:: + -- -example: Warning +type: alias -The log level. +alias to: log.level -- *`mysql.error.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- @@ -6372,30 +6370,6 @@ Contains fields from the MySQL slow logs. -*`mysql.slowlog.user`*:: -+ --- -The MySQL user that created the query. - - --- - -*`mysql.slowlog.host`*:: -+ --- -The host from where the user that created the query logged in. - - --- - -*`mysql.slowlog.ip`*:: -+ --- -The IP address from where the user that created the query logged in. - - --- - *`mysql.slowlog.query_time.sec`*:: + -- @@ -6462,6 +6436,33 @@ type: long The connection ID for the query. +-- + +*`mysql.slowlog.user`*:: ++ +-- +type: alias + +alias to: user.name + +-- + +*`mysql.slowlog.host`*:: ++ +-- +type: alias + +alias to: source.domain + +-- + +*`mysql.slowlog.ip`*:: ++ +-- +type: alias + +alias to: source.ip + -- [[exported-fields-netflow]] diff --git a/filebeat/module/mysql/error/_meta/fields.yml b/filebeat/module/mysql/error/_meta/fields.yml index c878689f968..39a7949da7d 100644 --- a/filebeat/module/mysql/error/_meta/fields.yml +++ b/filebeat/module/mysql/error/_meta/fields.yml @@ -6,16 +6,16 @@ - name: timestamp description: > The timestamp from the log line. + - name: thread_id - type: long - description: > - As of MySQL 5.7.2, this is the thread id. For MySQL versions prior to 5.7.2, this - field contains the process id. + type: alias + path: process.thread.id + migration: true - name: level - example: "Warning" - description: - The log level. + type: alias + path: log.level + migration: true - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json index 911302fde57..a541ed29aa0 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.json +++ b/filebeat/module/mysql/error/ingest/pipeline.json @@ -1,35 +1,23 @@ { - "description": "Pipeline for parsing MySQL error logs. It currently uses message1 and message2 to workaround bug https://github.com/elastic/elasticsearch/issues/22117", + "description": "Pipeline for parsing MySQL error logs", "processors": [{ "grok": { "field": "message", "patterns": [ - "%{LOCALDATETIME:mysql.error.timestamp} (\\[%{DATA:mysql.error.level}\\] )?%{GREEDYDATA:mysql.error.message}", - "%{DATA:mysql.error.timestamp} %{NUMBER:mysql.error.thread_id} \\[%{DATA:mysql.error.level}\\] %{GREEDYDATA:mysql.error.message1}", - "%{GREEDYDATA:mysql.error.message2}" + "%{LOCALDATETIME:mysql.error.timestamp} (\\[%{DATA:log.level}\\] )?%{GREEDYDATA:message}", + "%{DATA:mysql.error.timestamp} %{NUMBER:process.thread.id:long} \\[%{DATA:log.level}\\] %{GREEDYDATA:message}", + "%{GREEDYDATA:message}" ], "ignore_missing": true, "pattern_definitions": { - "LOCALDATETIME": "[0-9]+ %{TIME}", - "GREEDYDATA1": ".*" + "LOCALDATETIME": "[0-9]+ %{TIME}" } } }, { - "remove": { - "field": "message" - } - }, { - "rename": { - "field": "mysql.error.message1", - "target_field": "mysql.error.message", - "ignore_failure": true - } - }, { - "rename": { - "field": "mysql.error.message2", - "target_field": "mysql.error.message", - "ignore_failure": true - } + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } }, { "date": { "field": "mysql.error.timestamp", diff --git a/filebeat/module/mysql/error/test/error.log b/filebeat/module/mysql/error/test/error.log new file mode 100644 index 00000000000..37cd1af5c78 --- /dev/null +++ b/filebeat/module/mysql/error/test/error.log @@ -0,0 +1,12 @@ +161209 13:08:33 mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql +2016-12-09T12:08:33.335060Z 0 [Warning] TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details). +2016-12-09T12:08:33.335892Z 0 [Warning] Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path. +2016-12-09T12:08:33.336610Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 61571 ... +2016-12-09T12:08:33.345527Z 0 [Warning] Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive +2016-12-09T12:08:33.351596Z 0 [Note] InnoDB: Mutexes and rw_locks use GCC atomic builtins +2016-12-09T12:08:33.784722Z 0 [Note] /usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections. +Version: '5.7.10' socket: '/tmp/mysql.sock' port: 3306 Homebrew +2016-12-09T22:21:02.443689Z 0 [Note] InnoDB: page_cleaner: 1000ms intended loop took 772568ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.) +161209 14:18:50 [Warning] Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead. +161209 14:18:50 [Note] Plugin 'FEDERATED' is disabled. +161209 14:18:50 InnoDB: The InnoDB memory heap is disabled diff --git a/filebeat/module/mysql/error/test/error.log-expected.json b/filebeat/module/mysql/error/test/error.log-expected.json new file mode 100644 index 00000000000..86c5cd0193e --- /dev/null +++ b/filebeat/module/mysql/error/test/error.log-expected.json @@ -0,0 +1,148 @@ +[ + { + "@timestamp": "2016-12-09T13:08:33.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.offset": 0, + "message": "mysqld_safe Starting mysqld daemon with databases from /usr/local/var/mysql", + "mysql.error.timestamp": "161209 13:08:33" + }, + { + "@timestamp": "2016-12-09T12:08:33.335Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 92, + "message": "TIMESTAMP with implicit DEFAULT value is deprecated. Please use --explicit_defaults_for_timestamp server option (see documentation for more details).", + "mysql.error.timestamp": "2016-12-09T12:08:33.335060Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T12:08:33.335Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 282, + "message": "Insecure configuration for --secure-file-priv: Current value does not restrict location of generated files. Consider setting it to a valid, non-empty path.", + "mysql.error.timestamp": "2016-12-09T12:08:33.335892Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T12:08:33.336Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 478, + "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld (mysqld 5.7.10) starting as process 61571 ...", + "mysql.error.timestamp": "2016-12-09T12:08:33.336610Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T12:08:33.345Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 603, + "message": "Setting lower_case_table_names=2 because file system for /usr/local/var/mysql/ is case insensitive", + "mysql.error.timestamp": "2016-12-09T12:08:33.345527Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T12:08:33.351Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 742, + "message": "InnoDB: Mutexes and rw_locks use GCC atomic builtins", + "mysql.error.timestamp": "2016-12-09T12:08:33.351596Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T12:08:33.784Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 832, + "message": "/usr/local/Cellar/mysql/5.7.10/bin/mysqld: ready for connections.", + "mysql.error.timestamp": "2016-12-09T12:08:33.784722Z", + "process.thread.id": 0 + }, + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.offset": 935, + "message": "Version: '5.7.10' socket: '/tmp/mysql.sock' port: 3306 Homebrew" + }, + { + "@timestamp": "2016-12-09T22:21:02.443Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 1002, + "message": "InnoDB: page_cleaner: 1000ms intended loop took 772568ms. The settings might not be optimal. (flushed=0 and evicted=0, during the time.)", + "mysql.error.timestamp": "2016-12-09T22:21:02.443689Z", + "process.thread.id": 0 + }, + { + "@timestamp": "2016-12-09T14:18:50.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Warning", + "log.offset": 1176, + "message": "Using unique option prefix myisam-recover instead of myisam-recover-options is deprecated and will be removed in a future release. Please use the full name instead.", + "mysql.error.timestamp": "161209 14:18:50" + }, + { + "@timestamp": "2016-12-09T14:18:50.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.level": "Note", + "log.offset": 1367, + "message": "Plugin 'FEDERATED' is disabled.", + "mysql.error.timestamp": "161209 14:18:50" + }, + { + "@timestamp": "2016-12-09T14:18:50.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "mysql.error", + "event.module": "mysql", + "fileset.name": "error", + "input.type": "log", + "log.offset": 1422, + "message": "InnoDB: The InnoDB memory heap is disabled", + "mysql.error.timestamp": "161209 14:18:50" + } +] \ No newline at end of file diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go index ba82afba23b..ab5fe57e6c7 100644 --- a/filebeat/module/mysql/fields.go +++ b/filebeat/module/mysql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy0lcFu2zAMhu95CqLn1ocBw4AcBgzbChRYgQ0tsGOq2LQtRCZdUk6atx8op02WqtmStjpa0v9/pCnyAha4nkK31vswAYg+BpzC2fX65tePswlAhVqK76NnmsLnCQDANVdDQKhZoHeinhqILUK6AoEbqH1ALSYA2rLEWclU+2YKUQacANQeQ6XTJHUB5Drc2tuK6x6n0AgP/eZLhsHWZRKCWrh7CcDWrt+uJ4qwPH3N+R7wtvWVKTpPunHYB0n6hvNEkqPZJYq+Q42u6//aPchg67bF7dUthSUieMIi79UKumrmqz25MQuBqTkO4osC15vIPxafig/nEFuv4DXBjHbgqwIuWTbnlijqmRR68SwQeffmM4eUOSgfk26qvXCJqiabDTLgEsOeED64rk81/tsJeWrODgSazXVKrCnnTTtUdQ1m8xrxIR7/cwM3DVaPwsWzStbAq8DNO9ayORxVyoOiHB/oaGZ3IbYuQinoIlYJ5H5AWecz3rKekFW7Nca5alEwmRywfvwLnvIQ/oRXe/UTXFWJlfDbgaRDM2sIhWKZLcM6sDshY5GjC6nV7NBE5sU5eALFkqnSc3AKbrSw0dCzpwg0dHOUF14pl4t34XUdDxStL+0xr5y3lNoEGztlubDuM0dwS+eDmwcsTOGZ6tKFAa2nvSZe4ZXOFGk/olNbr4U6+lmopg6CcRDCCubrfz2eRGNN0RO+1TDIEGnp6P+AXpqCr4EZyD/szMjoFkjbHnd38/12u3t3CC5tHe+f+ucB2Tcbw2ZWMhGWdgKuvj0V+cb9TwAAAP//tXis5A==" + return "eJy0lMFu2zwMgO95CqLn1g/gw3/5twED1sPQ3lPGpm0hMumKUlK//SA7izNHyep05dEK+X1UKD7Alvoc2l5f7QrAG28ph7vH/unnj7sVQElaONN5I5zDfysAgEcpgyWoxEGHTg3X4BuCIQWs1FAZS5qtALQR59eFcGXqHLwLtAKoDNlS86HUAzC2NOFj+L6jHGonoTt8STjE+DYUgspJe0kgxinvlEnOiTt+TXGvsGP8L+zRsB4Ic5GhftQ5mqRsTo28aUk9tt0fp1cdYjw3NKVOFvEirGGa6Glo4wjLtSlndcfrQGtQZycd+iaHzklBqtmYn53lt6Z2OCof/vhztqUd2YVcK3WWynsPryVVrGkhMZ11ifebpVb2VupPHLBIWDRfr4Fcv46jkikVyVuorKC/YfzEox2GcPAbQOBFtvdgGJQK4VLvARVwRMSl0YlhDxzaDbksPR9SbD/FF1sJ7EGqufMejady2G3jGyq24AU2BLhDY3FjKYsVzqru0AYCox/q18le10o872js1QrXy1sdebHVWB0c+eCYStj0U9tXbOgNW8OU3g7/xEgL5PcJXdqPH5EJbN5OtqfHLfH00F6evj5Ppy/X5Iaj5fzhEV8pe2Ev39ZsIcxUxF/A9y/HIZ/RkxpByS1cmzEli9k3LOpGNP0GLuNUgisoK6VFwzcgTXqq/go8y5vDfgUAAP//9N14WQ==" } diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml index c785d9bd4b1..c6b4a2ac5a2 100644 --- a/filebeat/module/mysql/slowlog/_meta/fields.yml +++ b/filebeat/module/mysql/slowlog/_meta/fields.yml @@ -3,15 +3,6 @@ description: > Contains fields from the MySQL slow logs. fields: - - name: user - description: > - The MySQL user that created the query. - - name: host - description: > - The host from where the user that created the query logged in. - - name: ip - description: > - The IP address from where the user that created the query logged in. - name: query_time.sec type: float description: > @@ -40,3 +31,16 @@ type: long description: > The connection ID for the query. + + - name: user + type: alias + path: user.name + migration: true + - name: host + type: alias + path: source.domain + migration: true + - name: ip + type: alias + path: source.ip + migration: true diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 78e5e031e8d..c8287e947c7 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "^# User@Host: %{USER:mysql.slowlog.user}(\\[[^\\]]+\\])? @ (%{HOSTNAME:mysql.slowlog.host})? \\[(%{IP:mysql.slowlog.ip})?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id})?\n# Query_time: %{NUMBER:mysql.slowlog.query_time.sec}\\s* Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec}\\s* Rows_sent: %{NUMBER:mysql.slowlog.rows_sent}\\s* Rows_examined: %{NUMBER:mysql.slowlog.rows_examined}\n(SET timestamp=%{NUMBER:mysql.slowlog.timestamp};\n)?%{GREEDYMULTILINE:mysql.slowlog.query}" + "^# User@Host: %{USER:user.name}(\\[[^\\]]+\\])? @ (%{HOSTNAME:source.domain})? \\[(%{IP:source.ip})?\\](\\s*Id:\\s* %{NUMBER:mysql.slowlog.id:long})?\n# Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}\\s* Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}\\s* Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}\\s* Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}\n(SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n)?%{GREEDYMULTILINE:mysql.slowlog.query}" ], "pattern_definitions" : { "GREEDYMULTILINE" : "(.|\n)*" @@ -15,6 +15,12 @@ "remove":{ "field": "message" } + }, { + "script":{ + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.mysql.slowlog.query_time.sec * 1000000) * 1000", + "if": "ctx.mysql.slowlog.query_time?.sec != null" + } }, { "date": { "field": "mysql.slowlog.timestamp", diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log index a888e66bc9a..506ec108f87 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log @@ -9,7 +9,7 @@ select sleep(15); SET timestamp=1533630467; SELECT count(*) FROM mysql.user WHERE user='root' and password=''; # Time: 2018-08-07T16:27:47.169604+08:00 -# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10997316 +# User@Host: appuser[appuser] @ apphost [1.1.1.1] Id: 10997316 # Query_time: 4.071491 Lock_time: 0.000212 Rows_sent: 1000 Rows_examined: 1489615 SET timestamp=1533630467; SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR ";") as mca_guid @@ -24,7 +24,7 @@ SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_o ORDER BY mcu.mcu_order ASC LIMIT 1000; # Time: 2018-08-07T16:27:47.169604+08:00 -# User@Host: apphost[apphost] @ apphost [1.1.1.1] Id: 10999834 +# User@Host: appuser[appuser] @ apphost [1.1.1.1] Id: 10999834 # Query_time: 10.346539 Lock_time: 0.000036 Rows_sent: 0 Rows_examined: 4751313 SET timestamp=1533630467; call load_stats(1, '2017-04-28 00:00:00'); diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json index ff64b75840f..093232beaac 100644 --- a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json +++ b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json @@ -3,6 +3,7 @@ "@timestamp": "2018-08-07T08:27:47.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "mysql.slowlog", + "event.duration": 15000223000, "event.module": "mysql", "fileset.name": "slowlog", "input.type": "log", @@ -10,20 +11,21 @@ "multiline" ], "log.offset": 41, - "mysql.slowlog.id": "7234", - "mysql.slowlog.ip": "218.76.8.37", - "mysql.slowlog.lock_time.sec": "0.000000", + "mysql.slowlog.id": 7234, + "mysql.slowlog.lock_time.sec": 0.0, "mysql.slowlog.query": "select sleep(15);", - "mysql.slowlog.query_time.sec": "15.000223", - "mysql.slowlog.rows_examined": "0", - "mysql.slowlog.rows_sent": "1", - "mysql.slowlog.timestamp": "1533630467", - "mysql.slowlog.user": "root" + "mysql.slowlog.query_time.sec": 15.000223, + "mysql.slowlog.rows_examined": 0, + "mysql.slowlog.rows_sent": 1, + "mysql.slowlog.timestamp": 1533630467, + "source.ip": "218.76.8.37", + "user.name": "root" }, { "@timestamp": "2018-08-07T08:27:47.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "mysql.slowlog", + "event.duration": 153000, "event.module": "mysql", "fileset.name": "slowlog", "input.type": "log", @@ -31,19 +33,20 @@ "multiline" ], "log.offset": 254, - "mysql.slowlog.host": "localhost", - "mysql.slowlog.lock_time.sec": "0.000061", + "mysql.slowlog.lock_time.sec": 6.1e-05, "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';", - "mysql.slowlog.query_time.sec": "0.000153", - "mysql.slowlog.rows_examined": "5", - "mysql.slowlog.rows_sent": "1", - "mysql.slowlog.timestamp": "1533630467", - "mysql.slowlog.user": "debian-sys-maint" + "mysql.slowlog.query_time.sec": 0.000153, + "mysql.slowlog.rows_examined": 5, + "mysql.slowlog.rows_sent": 1, + "mysql.slowlog.timestamp": 1533630467, + "source.domain": "localhost", + "user.name": "debian-sys-maint" }, { "@timestamp": "2018-08-07T08:27:47.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "mysql.slowlog", + "event.duration": 4071491000, "event.module": "mysql", "fileset.name": "slowlog", "input.type": "log", @@ -51,21 +54,22 @@ "multiline" ], "log.offset": 526, - "mysql.slowlog.host": "apphost", - "mysql.slowlog.id": "10997316", - "mysql.slowlog.ip": "1.1.1.1", - "mysql.slowlog.lock_time.sec": "0.000212", + "mysql.slowlog.id": 10997316, + "mysql.slowlog.lock_time.sec": 0.000212, "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n WHERE cus.cus_guid = mcu.cus_guid\n AND cus.pro_code = 'CYB'\n AND cus.cus_offline = 0\n AND mca.cus_guid = cus.cus_guid\n AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n AND mcu.mcu_crawlelements IS NOT NULL\n GROUP BY mcu.mcu_guid\n ORDER BY mcu.mcu_order ASC\n LIMIT 1000;", - "mysql.slowlog.query_time.sec": "4.071491", - "mysql.slowlog.rows_examined": "1489615", - "mysql.slowlog.rows_sent": "1000", - "mysql.slowlog.timestamp": "1533630467", - "mysql.slowlog.user": "apphost" + "mysql.slowlog.query_time.sec": 4.071491, + "mysql.slowlog.rows_examined": 1489615, + "mysql.slowlog.rows_sent": 1000, + "mysql.slowlog.timestamp": 1533630467, + "source.domain": "apphost", + "source.ip": "1.1.1.1", + "user.name": "appuser" }, { "@timestamp": "2018-08-07T08:27:47.000Z", "ecs.version": "1.0.0-beta2", "event.dataset": "mysql.slowlog", + "event.duration": 10346539000, "event.module": "mysql", "fileset.name": "slowlog", "input.type": "log", @@ -73,15 +77,15 @@ "multiline" ], "log.offset": 1438, - "mysql.slowlog.host": "apphost", - "mysql.slowlog.id": "10999834", - "mysql.slowlog.ip": "1.1.1.1", - "mysql.slowlog.lock_time.sec": "0.000036", + "mysql.slowlog.id": 10999834, + "mysql.slowlog.lock_time.sec": 3.6e-05, "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');", - "mysql.slowlog.query_time.sec": "10.346539", - "mysql.slowlog.rows_examined": "4751313", - "mysql.slowlog.rows_sent": "0", - "mysql.slowlog.timestamp": "1533630467", - "mysql.slowlog.user": "apphost" + "mysql.slowlog.query_time.sec": 10.346539, + "mysql.slowlog.rows_examined": 4751313, + "mysql.slowlog.rows_sent": 0, + "mysql.slowlog.timestamp": 1533630467, + "source.domain": "apphost", + "source.ip": "1.1.1.1", + "user.name": "appuser" } ] \ No newline at end of file