From 53f41c17cb46f30389f702e18d59072e70aed70d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 18 Dec 2018 16:14:27 -0700 Subject: [PATCH 1/2] Support different timestamp format in postgresql module --- CHANGELOG.asciidoc | 1 + .../postgresql/log/ingest/pipeline.json | 2 +- .../log/test/postgresql-9.6-new-timestamp.log | 5 ++ ...gresql-9.6-new-timestamp.log-expected.json | 76 +++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log create mode 100644 filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 696b0f2debc..a5e7e9c0071 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -321,6 +321,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346] - Mark the TCP and UDP input as GA. {pull}8125[8125] - Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562] +- Support different timestamp format in postgresql module. {issue}9494[9494] *Heartbeat* diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json index 398b1d95405..aa6d8309482 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.json +++ b/filebeat/module/postgresql/log/ingest/pipeline.json @@ -20,7 +20,7 @@ "field": "postgresql.log.timestamp", "target_field": "@timestamp", "formats": [ - "yyyy-MM-dd HH:mm:ss.SSS" + "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ], "ignore_failure": true } diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log new file mode 100644 index 00000000000..2042fb9f0a6 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log @@ -0,0 +1,5 @@ +2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started +2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart) +2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter "max_wal_size". +2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up +2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json new file mode 100644 index 00000000000..3a3fb62a734 --- /dev/null +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json @@ -0,0 +1,76 @@ +[ + { + "@timestamp": "2017-07-31T13:36:43.000Z", + "event.dataset": "log", + "event.module": "postgresql", + "input.type": "log", + "log.offset": 0, + "message": "2017-07-31 13:36:43 EST [835-1] LOG: autovacuum launcher started", + "postgresql.log.core_id": "1", + "postgresql.log.level": "LOG", + "postgresql.log.message": "autovacuum launcher started", + "postgresql.log.thread_id": "835", + "postgresql.log.timestamp": "2017-07-31 13:36:43", + "postgresql.log.timezone": "EST" + }, + { + "@timestamp": "2017-07-31T13:36:44.000Z", + "event.dataset": "log", + "event.module": "postgresql", + "input.type": "log", + "log.offset": 66, + "message": "2017-07-31 13:36:44 EST [832-1] LOG: checkpoints are occurring too frequently (25 seconds apart)", + "postgresql.log.core_id": "1", + "postgresql.log.level": "LOG", + "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", + "postgresql.log.thread_id": "832", + "postgresql.log.timestamp": "2017-07-31 13:36:44", + "postgresql.log.timezone": "EST" + }, + { + "@timestamp": "2017-07-31T13:46:02.000Z", + "event.dataset": "log", + "event.module": "postgresql", + "input.type": "log", + "log.offset": 164, + "message": "2017-07-31 13:46:02 EST [832-2] HINT: Consider increasing the configuration parameter \"max_wal_size\".", + "postgresql.log.core_id": "2", + "postgresql.log.level": "HINT", + "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", + "postgresql.log.thread_id": "832", + "postgresql.log.timestamp": "2017-07-31 13:46:02", + "postgresql.log.timezone": "EST" + }, + { + "@timestamp": "2017-07-31T13:46:23.000Z", + "event.dataset": "log", + "event.module": "postgresql", + "input.type": "log", + "log.offset": 267, + "message": "2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL: the database system is starting up", + "postgresql.log.core_id": "1", + "postgresql.log.database": "postgres", + "postgresql.log.level": "FATAL", + "postgresql.log.message": "the database system is starting up", + "postgresql.log.thread_id": "768", + "postgresql.log.timestamp": "2017-07-31 13:46:23", + "postgresql.log.timezone": "EST", + "postgresql.log.user": "postgres" + }, + { + "@timestamp": "2017-07-31T13:46:55.000Z", + "event.dataset": "log", + "event.module": "postgresql", + "input.type": "log", + "log.offset": 360, + "message": "2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL: the database system is starting up", + "postgresql.log.core_id": "1", + "postgresql.log.database": "postgres", + "postgresql.log.level": "FATAL", + "postgresql.log.message": "the database system is starting up", + "postgresql.log.thread_id": "771", + "postgresql.log.timestamp": "2017-07-31 13:46:55", + "postgresql.log.timezone": "EST", + "postgresql.log.user": "postgres" + } +] \ No newline at end of file From 4835c1c258bfd35fdb0c1ddd138b0cdac4478cdd Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 18 Dec 2018 16:18:30 -0700 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a5e7e9c0071..ec35526e208 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -321,7 +321,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346] - Mark the TCP and UDP input as GA. {pull}8125[8125] - Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562] -- Support different timestamp format in postgresql module. {issue}9494[9494] +- Support different timestamp format in postgresql module. {issue}9494[9494] {pull}9650[9650] *Heartbeat*