From 4f3f6350b7fc9d5deefb59afbdf194961f8f8ab3 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 21 Aug 2019 13:08:02 -0600 Subject: [PATCH 1/5] Add timezone information in Apache Error metricset --- filebeat/module/apache/error/config/error.yml | 3 +++ .../module/apache/error/ingest/pipeline.json | 20 +++++++++++++++++++ .../test/darwin-2.4.23.log-expected.json | 2 ++ .../apache/error/test/test.log-expected.json | 4 ++++ .../test/ubuntu-2.2.22.log-expected.json | 7 +++++++ 5 files changed, 36 insertions(+) diff --git a/filebeat/module/apache/error/config/error.yml b/filebeat/module/apache/error/config/error.yml index 0afd17317d4..d96242ac040 100644 --- a/filebeat/module/apache/error/config/error.yml +++ b/filebeat/module/apache/error/config/error.yml @@ -4,3 +4,6 @@ paths: - {{$path}} {{ end }} exclude_files: [".gz$"] + +processors: +- add_locale: ~ diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json index 87cd1d64804..ce12bc78fe2 100644 --- a/filebeat/module/apache/error/ingest/pipeline.json +++ b/filebeat/module/apache/error/ingest/pipeline.json @@ -16,15 +16,35 @@ }, { "date": { + "if": "ctx.event.timezone != null", "field": "apache.error.timestamp", "target_field": "@timestamp", "formats": [ "EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy" ], + "timezone": "{{ event.timezone }}", "ignore_failure": true } }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "@timestamp", + "formats": [ + "ISO8601" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, { "remove": { "field": "apache.error.timestamp", diff --git a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json index 244f4c43132..3098d1e6291 100644 --- a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json @@ -4,6 +4,7 @@ "apache.error.module": "mpm_prefork", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -17,6 +18,7 @@ "apache.error.module": "core", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "notice", diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index 3160d461d60..598ec3b9020 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -3,6 +3,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -17,6 +18,7 @@ "apache.error.module": "core", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -30,6 +32,7 @@ "apache.error.module": "core", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -55,6 +58,7 @@ "apache.error.module": "include", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "warn", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 5b8eaf2fe30..70761f7c590 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -3,6 +3,7 @@ "@timestamp": "2016-12-26T16:17:53.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "notice", @@ -14,6 +15,7 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -27,6 +29,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -40,6 +43,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -53,6 +57,7 @@ "@timestamp": "2016-12-26T16:22:10.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -66,6 +71,7 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -79,6 +85,7 @@ "@timestamp": "2016-12-26T16:22:17.000Z", "event.dataset": "apache.error", "event.module": "apache", + "event.timezone": "+00:00", "fileset.name": "error", "input.type": "log", "log.level": "error", From b3800d7b3deb54237d00cb3222975b07f01c3668 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 21 Aug 2019 13:21:50 -0600 Subject: [PATCH 2/5] Update changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f4f2e40050e..ea2d809d1ad 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Allow path variables to be used in files loaded from modules.d. {issue}13184[13184] - Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303] - Change iis url.path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] +- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] *Heartbeat* From 6fbf9523e1cdff60dde767402b8814b83c93779c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 21 Aug 2019 20:29:59 -0600 Subject: [PATCH 3/5] fix duplicate date pipeline --- filebeat/module/apache/error/ingest/pipeline.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json index ce12bc78fe2..fdcc737575b 100644 --- a/filebeat/module/apache/error/ingest/pipeline.json +++ b/filebeat/module/apache/error/ingest/pipeline.json @@ -16,14 +16,12 @@ }, { "date": { - "if": "ctx.event.timezone != null", "field": "apache.error.timestamp", "target_field": "@timestamp", "formats": [ "EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy" ], - "timezone": "{{ event.timezone }}", "ignore_failure": true } }, From 4f06aad2e7e53056baee3c470b9e680fd254f443 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 22 Aug 2019 19:19:42 -0600 Subject: [PATCH 4/5] Add timezone-support link --- filebeat/docs/modules/apache.asciidoc | 2 ++ filebeat/module/apache/_meta/docs.asciidoc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/filebeat/docs/modules/apache.asciidoc b/filebeat/docs/modules/apache.asciidoc index 0834798194d..e674e0ad968 100644 --- a/filebeat/docs/modules/apache.asciidoc +++ b/filebeat/docs/modules/apache.asciidoc @@ -71,6 +71,8 @@ include::../include/var-paths.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + :has-dashboards!: :fileset_ex!: diff --git a/filebeat/module/apache/_meta/docs.asciidoc b/filebeat/module/apache/_meta/docs.asciidoc index f19b3880258..ef2f4196230 100644 --- a/filebeat/module/apache/_meta/docs.asciidoc +++ b/filebeat/module/apache/_meta/docs.asciidoc @@ -66,6 +66,8 @@ include::../include/var-paths.asciidoc[] include::../include/var-paths.asciidoc[] +include::../include/timezone-support.asciidoc[] + :has-dashboards!: :fileset_ex!: From a82fca39f94f2d564a0dd0cd810b6097bb01f2e8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 23 Aug 2019 14:19:51 -0600 Subject: [PATCH 5/5] Fix date pipeline --- filebeat/module/apache/error/ingest/pipeline.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/filebeat/module/apache/error/ingest/pipeline.json b/filebeat/module/apache/error/ingest/pipeline.json index fdcc737575b..8a1268780b1 100644 --- a/filebeat/module/apache/error/ingest/pipeline.json +++ b/filebeat/module/apache/error/ingest/pipeline.json @@ -28,9 +28,11 @@ { "date": { "if": "ctx.event.timezone != null", - "field": "@timestamp", + "field": "apache.error.timestamp", + "target_field": "@timestamp", "formats": [ - "ISO8601" + "EEE MMM dd H:m:s yyyy", + "EEE MMM dd H:m:s.SSSSSS yyyy" ], "timezone": "{{ event.timezone }}", "on_failure": [