Remove event.timezone from events from some json logs (#13918)

Filebeat modules for Elasticsearch and Logstash support two different log formats, the JSON one contains timezones, so it doesn't need the `event.timezone` added by `add_locale` for date parsing. Also having this added `event.timezone` can be inconsistent in some cases as it may be different to the timezone of the logs parsed. Don't add this field when the log message is in JSON format.
elastic · Oct 4, 2019 · 1af0403 · 1af0403
1 parent 4ec2854
commit 1af0403
Show file tree

Hide file tree

Showing 19 changed files with 13 additions and 122 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -329,6 +329,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support to set the document id in the json reader. {pull}5844[5844]
 - Add input httpjson. {issue}13545[13545] {pull}13546[13546]
 - Filebeat Netflow input: Remove beta label. {pull}13858[13858]
+- Remove `event.timezone` from events that don't need it in some modules that support log formats with and without timezones. {pull}13918[13918]
 
 *Heartbeat*
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]

diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -6,4 +6,5 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-- add_locale: ~
+# Locale for timezone is only needed in non-json logs
+- add_locale.when.not.regexp.message: "^{"
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-730.log-expected.json
@@ -18,7 +18,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 0,
@@ -48,7 +47,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 423,
@@ -78,7 +76,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 846,
@@ -107,7 +104,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1269,
@@ -136,7 +132,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1706,
@@ -162,7 +157,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 2170,
@@ -188,7 +182,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 2576,
@@ -217,7 +210,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 2984,
@@ -246,7 +238,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 3402,
@@ -272,7 +263,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 3823,

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
@@ -8,7 +8,6 @@
         "event.action": "anonymous_access_denied",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "http.request.method": "GET",
         "input.type": "log",
@@ -29,7 +28,6 @@
         "event.action": "authentication_failed",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "http.request.method": "GET",
         "input.type": "log",

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json
@@ -7,7 +7,6 @@
         "event.action": "authentication_failed",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 0,
@@ -27,7 +26,6 @@
         "event.action": "authentication_failed",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 274,
@@ -53,7 +51,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 558,
@@ -78,7 +75,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 941,
@@ -103,7 +99,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1309,
@@ -131,7 +126,6 @@
         "event.action": "access_granted",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "input.type": "log",
         "log.offset": 1676,
@@ -153,7 +147,6 @@
         "event.action": "authentication_success",
         "event.dataset": "elasticsearch.audit",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "audit",
         "http.request.body.content": "\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n",
         "http.request.method": "GET",

diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml
@@ -10,4 +10,5 @@ multiline:
   match: after
 
 processors:
-- add_locale: ~
+# Locale for timezone is only needed in non-json logs
+- add_locale.when.not.regexp.message: "^{"
diff --git a/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json
@@ -8,7 +8,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -25,7 +24,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -42,7 +40,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -59,7 +56,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -76,7 +72,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -93,7 +88,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -110,7 +104,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -127,7 +120,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -144,7 +136,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -161,7 +152,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -178,7 +168,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -195,7 +184,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",
@@ -212,7 +200,6 @@
         "elasticsearch.node.name": "es1_1",
         "event.dataset": "elasticsearch.deprecation",
         "event.module": "elasticsearch",
-        "event.timezone": "-02:00",
         "fileset.name": "deprecation",
         "input.type": "log",
         "log.level": "WARN",

diff --git a/filebeat/module/elasticsearch/server/config/log.yml b/filebeat/module/elasticsearch/server/config/log.yml
@@ -10,5 +10,5 @@ multiline:
   match: after
 
 processors:
-- add_locale: ~
-
+# Locale for timezone is only needed in non-json logs
+- add_locale.when.not.regexp.message: "^{"