Fix filebeat elasticsearch module ingest timezone (#13367) (#13495)

(cherry picked from commit 31e9a1e) Co-authored-by: Kent Wang <pragkent@gmail.com>
elastic · Sep 5, 2019 · e8feb4e · e8feb4e
1 parent d9ca2d1
commit e8feb4e
Show file tree

Hide file tree

Showing 13 changed files with 129 additions and 76 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -79,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reduce memory usage if long lines are truncated to fit `max_bytes` limit. The line buffer is copied into a smaller buffer now. This allows the runtime to release unused memory earlier. {pull}11524[11524]
 - Fix filebeat autodiscover fileset hint for container input. {pull}13296[13296]
 - Fix timezone parsing of system module ingest pipelines. {pull}13308[13308]
+- Fix timezone parsing of elasticsearch module ingest pipelines. {pull}13367[13367]
 - Fix timezone parsing of nginx module ingest pipelines. {pull}13369[13369]
 - Fix incorrect field references in envoyproxy dashboard {issue}13420[13420] {pull}13421[13421]
 

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.json
@@ -190,6 +190,16 @@
                 "target_field": "log.level",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/audit/ingest/pipeline-plaintext.json
@@ -51,6 +51,28 @@
                 "field": "elasticsearch.audit.sub_action",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.audit.@timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.audit.@timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.audit.@timestamp"

diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.json
@@ -97,6 +97,16 @@
                 "field": "elasticsearch.deprecation.message",
                 "target_field": "message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.json
@@ -19,6 +19,28 @@
                     "\\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}*\\]\\[%{DATA:elasticsearch.component}%{SPACE}*\\] %{GREEDYMULTILINE:message}"
                 ]
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.deprecation.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.deprecation.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.deprecation.timestamp"

diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.json b/filebeat/module/elasticsearch/server/ingest/pipeline-json.json
@@ -107,6 +107,16 @@
             "remove": {
                 "field": "elasticsearch.server.message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/server/ingest/pipeline-plaintext.json
@@ -25,6 +25,28 @@
                     "%{LOG_HEADER}%{SPACE}((\\[%{INDEXNAME:elasticsearch.index.name}\\]|\\[%{INDEXNAME:elasticsearch.index.name}\\/%{DATA:elasticsearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}"
                 ]
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.server.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.json b/filebeat/module/elasticsearch/server/ingest/pipeline.json
@@ -54,25 +54,6 @@
                 "ignore_missing": true
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.server.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.server.timestamp"

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.json
@@ -103,6 +103,16 @@
             "remove": {
                 "field": "elasticsearch.slowlog.message"
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                "ignore_failure": true
+            }
         }
     ]
 }
diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-plaintext.json
@@ -20,6 +20,28 @@
                 "separator": ",",
                 "ignore_missing": true
             }
+        },
+        {
+            "date": {
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "ignore_failure": true
+            }
+        },
+        {
+            "date": {
+                "if": "ctx.event.timezone != null",
+                "field": "elasticsearch.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "yyyy-MM-dd'T'HH:mm:ss,SSS"
+                ],
+                "timezone": "{{ event.timezone }}",
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
+            }
         }
     ],
     "on_failure": [

diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.json
@@ -30,25 +30,6 @@
                 "name": "{< IngestPipeline "pipeline-json" >}"
             }
         },
-        {
-            "date": {
-                "field": "elasticsearch.slowlog.timestamp",
-                "target_field": "@timestamp",
-                "formats": [
-                    "ISO8601"
-                ],
-                "ignore_failure": true
-            }
-        },
-        {
-            "date": {
-                "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
-                "timezone": "{{ event.timezone }}",
-                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
-            }
-        },
         {
             "remove": {
                 "field": "elasticsearch.slowlog.timestamp"