Cherry-pick to 6.x: Add convert_timezone to nginx module (#10148) (#1…

…0219) * Add convert_timezone to nginx module (#10148) * Run make update for filebeat
elastic · Jan 21, 2019 · 563e422 · 563e422
1 parent 4222cf2
commit 563e422
Show file tree

Hide file tree

Showing 12 changed files with 61 additions and 2 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff]
 - Add support for ssl_request_log in apache2 module. {issue}8088[8088] {pull}9833[9833]
 - Add support for iis 7.5 log format. {issue}9753[9753] {pull}9967[9967]
 - Add support for MariaDB in the `slowlog` fileset of `mysql` module. {pull}9731[9731]
+- Add convert_timezone to nginx module. {issue}9839[9839] {pull}10148[10148]
 
 *Heartbeat*
 - Made monitors.d configuration part of the default config. {pull}9004[9004]

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -304,6 +304,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -316,6 +319,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
 #------------------------------- Osquery Module ------------------------------
 - module: osquery
   result:

diff --git a/filebeat/module/nginx/_meta/config.reference.yml b/filebeat/module/nginx/_meta/config.reference.yml
@@ -11,6 +11,9 @@
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -22,3 +25,6 @@
     # Input configuration (advanced). Any input configuration option
     # can be added under this section.
     #input:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
diff --git a/filebeat/module/nginx/_meta/config.yml b/filebeat/module/nginx/_meta/config.yml
@@ -7,10 +7,16 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
+
   # Error logs
   error:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml
@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -60,7 +60,9 @@
                 "target_field": "@timestamp",
                 "formats": [
                     "dd/MMM/YYYY:H:m:s Z"
-                ]
+                ],
+                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+                "ignore_failure": true
             }
         },
         {

diff --git a/filebeat/module/nginx/access/manifest.yml b/filebeat/module/nginx/access/manifest.yml
@@ -8,6 +8,13 @@ var:
       - /usr/local/var/log/nginx/access.log*
     os.windows:
       - c:/programdata/nginx/logs/*access.log*
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/default.json
 input: config/nginx-access.yml

diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml
@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/nginx/error/ingest/pipeline.json b/filebeat/module/nginx/error/ingest/pipeline.json
@@ -21,7 +21,9 @@
     "date": {
       "field": "nginx.error.time",
       "target_field": "@timestamp",
-      "formats": ["YYYY/MM/dd H:m:s"]
+      "formats": ["YYYY/MM/dd H:m:s"],
+      {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+      "ignore_failure": true
     }
   }, {
     "remove": {

diff --git a/filebeat/module/nginx/error/manifest.yml b/filebeat/module/nginx/error/manifest.yml
@@ -8,6 +8,13 @@ var:
       - /usr/local/var/log/nginx/error.log*
     os.windows:
       - c:/programdata/nginx/logs/error.log*
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/nginx-error.yml
diff --git a/filebeat/modules.d/nginx.yml.disabled b/filebeat/modules.d/nginx.yml.disabled
@@ -7,10 +7,16 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
+
   # Error logs
   error:
     enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: true
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -304,6 +304,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Error logs
   #error:
     #enabled: true
@@ -316,6 +319,9 @@ filebeat.modules:
     # can be added under this section.
     #input:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
 #-------------------------------- Osquery Module --------------------------------
 - module: osquery
   result: