diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0b42e13e71b..6b6076f65d5 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -181,6 +181,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Use `log.source.address` instead of `log.source.ip` for network input sources. {pull}9487[9487] - Rename many `redis.log.*` fields to map to ECS. {pull}9315[9315] - Rename many `icinga.*` fields to map to ECS. {pull}9294[9294] +- Rename many `postgresql.log.*` fields to map to ECS. {pull}9303[9303] *Metricbeat* diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index f95542a780d..d7d0cd80931 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -434,6 +434,28 @@ to: user_agent.original alias: true +## PostgreSQL module + +- from: postgresql.log.timezone + to: event.timezone + alias: true + +- from: postgresql.log.thread_id + to: process.pid + alias: true + +- from: postgresql.log.user + to: user.name + alias: true + +- from: postgresql.log.level + to: log.level + alias: true + +- from: postgresql.log.message + to: message + alias: true + ## Redis module - from: redis.log.pid diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index f116680db01..e1ac0b5811f 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -9892,88 +9892,87 @@ The timestamp from the log line. -- -*`postgresql.log.timezone`*:: +*`postgresql.log.core_id`*:: + -- -The timezone of timestamp. +type: long + +Core id -- -*`postgresql.log.thread_id`*:: +*`postgresql.log.database`*:: + -- -type: long - -Process id +example: mydb +Name of database -- -*`postgresql.log.core_id`*:: +*`postgresql.log.duration`*:: + -- -type: long +type: float -Core id +example: 30.0 +Duration of a query. -- -*`postgresql.log.user`*:: +*`postgresql.log.query`*:: + -- -example: admin +example: SELECT * FROM users; -Name of user +Query statement. -- -*`postgresql.log.database`*:: +*`postgresql.log.timezone`*:: + -- -example: mydb +type: alias -Name of database +alias to: event.timezone -- -*`postgresql.log.level`*:: +*`postgresql.log.thread_id`*:: + -- -example: FATAL +type: alias -The log level. +alias to: process.pid -- -*`postgresql.log.duration`*:: +*`postgresql.log.user`*:: + -- -type: float - -example: 30.0 +type: alias -Duration of a query. +alias to: user.name -- -*`postgresql.log.query`*:: +*`postgresql.log.level`*:: + -- -example: SELECT * FROM users; +type: alias -Query statement. +alias to: log.level -- *`postgresql.log.message`*:: + -- -type: text - -The logged message. +type: alias +alias to: message -- diff --git a/filebeat/module/postgresql/fields.go b/filebeat/module/postgresql/fields.go index 826ba953f82..559ee7adfb9 100644 --- a/filebeat/module/postgresql/fields.go +++ b/filebeat/module/postgresql/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJysk8Fq3DAQhu9+ip89FrIEenOhENLsadMmzd6DshprRSWNI8kl26cv8sa1o7VNHTJHifm/j5HmAr/oWKLmEJWn8GwKIOpoqMTq7nT4cL9dFYCksPe6jppdia8FANyybAyhYo9a+KCdQjwQ+j4YVqi0obAugHBgHx/37CqtSkTfUAFUmowMZZt3AScsZTap4rGmEspzU7+ejNicatPmofJsM5HWIdUQOcQaVoOgc+Ys9y15bgxd5RpDlagthShs/eZ2Fp9qd6C+tTdJeKMdrSdZf9jR+1CpE1z12AnIwZOQj1pmQacpG3ZqGf7O855CQBbY4fbs6eNg1+xpitQE8lkDvQhbtxskpNVuNQM7Q30Xth3nWWzHkyKKJxHy1+qZ9iif3oUcTf63G/SbzCRzc7W72i6C7rpvmXLHv4xsvEjNo49YGRZx0ufz5fpykc63V1aag8BzQ/44LtVeTXIfbrY31zt8wubnj9v2DcOXRR73KR4hikiWXBx3sBSCUPkPOM0l0ks+lv9YZMNKkeyC18XfAAAA///JvYrW" + return "eJyck0FP3DAQhe/5FU97rESE1Fsq9ULhBG0p3NGwmThW7YzxOKjbX185AXbJJivSOcZ67/sycc7wm3cVgmgykfXJFUCyyXGFzc/x4d3t9aYAatZttCFZ6Sp8LQDgRureMRqJCBTVdgapZexzcGLQWMdaFoC2EtPDVrrGmgop9lwAjWVXazX0naEjzxObPGkXuIKJ0oeXJzM241wNfWii+InI4JDnEHmIdWIOio6ZJ7nvyafW8DpTjUOVZD1rIh/enZ7E57lveR/dm2S8sx2Xs6ytRH6w9aRsfH0nnVmncCGRMWl7JdWU6JGUJyH+Qz4MN87v6sfNCd4R7Tt5hjTzzW/YPlKOz75h44TSotDn8/J8ldC3F1aWIjz1HHfzWx+OFrl3l9eXF/f4hKtfP27QK0f9ssrjNtdDEyX23KW9w+J1+yvd9MOMGyJnSScngVJbgZ9z9ULYWzOu4u1Xn+G2kaleunzL4BBly6plOEp+hJrXuRKYI2VO/wfO8TO7lTwnppzLfYTnWZXM2m85n5ry/gUAAP//D0KTig==" } diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml index 4e5a451eace..92b48ffb8c4 100644 --- a/filebeat/module/postgresql/log/_meta/fields.yml +++ b/filebeat/module/postgresql/log/_meta/fields.yml @@ -6,29 +6,14 @@ - name: timestamp description: > The timestamp from the log line. - - name: timezone - description: > - The timezone of timestamp. - - name: thread_id - type: long - description: > - Process id - name: core_id type: long description: > Core id - - name: user - example: "admin" - description: - Name of user - name: database example: "mydb" description: Name of database - - name: level - example: "FATAL" - description: - The log level. - name: duration type: float example: "30.0" @@ -38,7 +23,24 @@ example: "SELECT * FROM users;" description: Query statement. + + - name: timezone + type: alias + path: event.timezone + migration: true + - name: thread_id + type: alias + path: process.pid + migration: true + - name: user + type: alias + path: user.name + migration: true + - name: level + type: alias + path: log.level + migration: true - name: message - type: text - description: > - The logged message. + type: alias + path: message + migration: true diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 398b1d95405..d149f4be897 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -6,7 +6,7 @@ "field": "message", "ignore_missing": true, "patterns": [ - "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}(-%{BASE16FLOAT:postgresql.log.core_id})?\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})" + "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:postgresql.log.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions": { "LOCALDATETIME": "[-0-9]+ %{TIME}", @@ -24,6 +24,14 @@ ], "ignore_failure": true } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.postgresql.log.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.postgresql.log.containsKey('duration')" + } } ], "on_failure": [ diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index 4598f1e87b9..db026e9fcd3 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -3,288 +3,292 @@ "@timestamp": "2017-07-31T13:36:42.585Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 0, - "message": "2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST", - "postgresql.log.level": "LOG", - "postgresql.log.message": "database system was shut down at 2017-06-17 16:58:04 CEST", - "postgresql.log.thread_id": "4974", + "message": "database system was shut down at 2017-06-17 16:58:04 CEST", "postgresql.log.timestamp": "2017-07-31 13:36:42.585", - "postgresql.log.timezone": "CEST" + "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.605Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 100, - "message": "2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled", - "postgresql.log.level": "LOG", - "postgresql.log.message": "MultiXact member wraparound protections are now enabled", - "postgresql.log.thread_id": "4974", + "message": "MultiXact member wraparound protections are now enabled", "postgresql.log.timestamp": "2017-07-31 13:36:42.605", - "postgresql.log.timezone": "CEST" + "process.pid": 4974 }, { "@timestamp": "2017-07-31T13:36:42.615Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 198, - "message": "2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started", - "postgresql.log.level": "LOG", - "postgresql.log.message": "autovacuum launcher started", - "postgresql.log.thread_id": "4978", + "message": "autovacuum launcher started", "postgresql.log.timestamp": "2017-07-31 13:36:42.615", - "postgresql.log.timezone": "CEST" + "process.pid": 4978 }, { "@timestamp": "2017-07-31T13:36:42.616Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 268, - "message": "2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections", - "postgresql.log.level": "LOG", - "postgresql.log.message": "database system is ready to accept connections", - "postgresql.log.thread_id": "4973", + "message": "database system is ready to accept connections", "postgresql.log.timestamp": "2017-07-31 13:36:42.616", - "postgresql.log.timezone": "CEST" + "process.pid": 4973 }, { "@timestamp": "2017-07-31T13:36:42.956Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 357, - "message": "2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet", + "message": "incomplete startup packet", "postgresql.log.database": "unknown", - "postgresql.log.level": "LOG", - "postgresql.log.message": "incomplete startup packet", - "postgresql.log.thread_id": "4980", "postgresql.log.timestamp": "2017-07-31 13:36:42.956", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "unknown" + "process.pid": 4980, + "user.name": "unknown" }, { "@timestamp": "2017-07-31T13:36:43.557Z", "event.dataset": "log", + "event.duration": 37118000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 445, "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.118", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 37.118, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4983", "postgresql.log.timestamp": "2017-07-31 13:36:43.557", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 4983, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:44.104Z", "event.dataset": "log", + "event.duration": 2895000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 873, "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "2.895", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 2.895, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4986", "postgresql.log.timestamp": "2017-07-31 13:36:44.104", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 4986, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:44.642Z", "event.dataset": "log", + "event.duration": 2809000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 1300, "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "2.809", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 2.809, "postgresql.log.query": "SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", - "postgresql.log.thread_id": "4989", "postgresql.log.timestamp": "2017-07-31 13:36:44.642", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 4989, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:16.249Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "FATAL", "log.offset": 1727, - "message": "2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database \"users\" does not exist", + "message": "database \"users\" does not exist", "postgresql.log.database": "users", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"users\" does not exist", - "postgresql.log.thread_id": "5407", "postgresql.log.timestamp": "2017-07-31 13:39:16.249", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5407, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:17.945Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "FATAL", "log.offset": 1818, - "message": "2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database \"user\" does not exist", + "message": "database \"user\" does not exist", "postgresql.log.database": "user", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"user\" does not exist", - "postgresql.log.thread_id": "5500", "postgresql.log.timestamp": "2017-07-31 13:39:17.945", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5500, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:21.025Z", "event.dataset": "log", + "event.duration": 37598000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 1907, "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.598", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-07-31 13:39:21.025", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5404, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:31.619Z", "event.dataset": "log", + "event.duration": 9482000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 2620, "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": "9.482", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 9.482, "postgresql.log.query": "select * from clients;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:31.619", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:39:40.147Z", "event.dataset": "log", + "event.duration": 765000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 2733, "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "postgresql.log.database": "clients", - "postgresql.log.duration": "0.765", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 0.765, "postgresql.log.query": "select id from clients;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:39:40.147", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:40:54.310Z", "event.dataset": "log", + "event.duration": 26082001, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 2847, "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "postgresql.log.database": "clients", - "postgresql.log.duration": "26.082", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 26.082, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:40:54.310", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:43:22.645Z", "event.dataset": "log", + "event.duration": 36161999, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 3559, "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients", - "postgresql.log.duration": "36.162", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 36.162, "postgresql.log.query": "create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:43:22.645", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:02.670Z", "event.dataset": "log", + "event.duration": 10540000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 3751, "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "postgresql.log.database": "c$lients", - "postgresql.log.duration": "10.540", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 10.54, "postgresql.log.query": "insert into cats(name, toy, born) values('kate', 'ball', now());", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:02.670", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:23.016Z", "event.dataset": "log", + "event.duration": 5156000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 3908, "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "postgresql.log.database": "_clients$db", - "postgresql.log.duration": "5.156", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 5.156, "postgresql.log.query": "insert into cats(name, toy, born) values('frida', 'horse', now());", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", + "event.duration": 25871000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 4069, "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "postgresql.log.database": "clients_db", - "postgresql.log.duration": "25.871", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 25.871, "postgresql.log.query": "create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", - "postgresql.log.thread_id": "5502", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5502, + "user.name": "postgres" } ] \ No newline at end of file diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json index 223846890a4..c4c4d1b5536 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-multi-core.log-expected.json @@ -3,126 +3,120 @@ "@timestamp": "2017-04-03T22:32:14.322Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "LOG", "log.offset": 0, - "message": "2017-04-03 22:32:14.322 CEST [12975-1] [unknown]@[unknown] LOG: incomplete startup packet", - "postgresql.log.core_id": "1", + "message": "incomplete startup packet", + "postgresql.log.core_id": 1, "postgresql.log.database": "unknown", - "postgresql.log.level": "LOG", - "postgresql.log.message": "incomplete startup packet", - "postgresql.log.thread_id": "12975", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "unknown" + "process.pid": 12975, + "user.name": "unknown" }, { "@timestamp": "2017-04-03T22:32:14.322Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", + "log.level": "FATAL", "log.offset": 91, - "message": "2017-04-03 22:32:14.322 CEST [5404-1] postgres@user FATAL: database \"user\" does not exist", - "postgresql.log.core_id": "1", + "message": "database \"user\" does not exist", + "postgresql.log.core_id": 1, "postgresql.log.database": "user", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "database \"user\" does not exist", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:32:14.322", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5404, + "user.name": "postgres" }, { "@timestamp": "2017-04-03T22:35:22.389Z", "event.dataset": "log", + "event.duration": 37598000, "event.module": "postgresql", + "event.timezone": "CEST", "input.type": "log", "log.flags": [ "multiline" ], + "log.level": "LOG", "log.offset": 182, "message": "2017-04-03 22:35:22.389 CEST [5404-2] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.core_id": "2", + "postgresql.log.core_id": 2, "postgresql.log.database": "postgres", - "postgresql.log.duration": "37.598", - "postgresql.log.level": "LOG", + "postgresql.log.duration": 37.598, "postgresql.log.query": "SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", - "postgresql.log.thread_id": "5404", "postgresql.log.timestamp": "2017-04-03 22:35:22.389", - "postgresql.log.timezone": "CEST", - "postgresql.log.user": "postgres" + "process.pid": 5404, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:36:43.557Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "LOG", "log.offset": 897, - "message": "2017-07-31 13:36:43.557 EST [835-1] LOG: autovacuum launcher started", - "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "autovacuum launcher started", - "postgresql.log.thread_id": "835", + "message": "autovacuum launcher started", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:43.557", - "postgresql.log.timezone": "EST" + "process.pid": 835 }, { "@timestamp": "2017-07-31T13:36:44.227Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "LOG", "log.offset": 967, - "message": "2017-07-31 13:36:44.227 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart)", - "postgresql.log.core_id": "1", - "postgresql.log.level": "LOG", - "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", - "postgresql.log.thread_id": "832", + "message": "checkpoints are occurring too frequently (25 seconds apart)", + "postgresql.log.core_id": 1, "postgresql.log.timestamp": "2017-07-31 13:36:44.227", - "postgresql.log.timezone": "EST" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:02.670Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "HINT", "log.offset": 1069, - "message": "2017-07-31 13:46:02.670 EST [832-2] HINT: Consider increasing the configuration parameter \"max_wal_size\".", - "postgresql.log.core_id": "2", - "postgresql.log.level": "HINT", - "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", - "postgresql.log.thread_id": "832", + "message": "Consider increasing the configuration parameter \"max_wal_size\".", + "postgresql.log.core_id": 2, "postgresql.log.timestamp": "2017-07-31 13:46:02.670", - "postgresql.log.timezone": "EST" + "process.pid": 832 }, { "@timestamp": "2017-07-31T13:46:23.016Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "FATAL", "log.offset": 1176, - "message": "2017-07-31 13:46:23.016 EST [768-1] postgres@postgres FATAL: the database system is starting up", - "postgresql.log.core_id": "1", + "message": "the database system is starting up", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", - "postgresql.log.thread_id": "768", "postgresql.log.timestamp": "2017-07-31 13:46:23.016", - "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "process.pid": 768, + "user.name": "postgres" }, { "@timestamp": "2017-07-31T13:46:55.637Z", "event.dataset": "log", "event.module": "postgresql", + "event.timezone": "EST", "input.type": "log", + "log.level": "FATAL", "log.offset": 1273, - "message": "2017-07-31 13:46:55.637 EST [771-1] postgres@postgres FATAL: the database system is starting up", - "postgresql.log.core_id": "1", + "message": "the database system is starting up", + "postgresql.log.core_id": 1, "postgresql.log.database": "postgres", - "postgresql.log.level": "FATAL", - "postgresql.log.message": "the database system is starting up", - "postgresql.log.thread_id": "771", "postgresql.log.timestamp": "2017-07-31 13:46:55.637", - "postgresql.log.timezone": "EST", - "postgresql.log.user": "postgres" + "process.pid": 771, + "user.name": "postgres" } ] \ No newline at end of file