From 9f473a9b7e47d99df4046cb67af6f839d46b9abc Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Mon, 20 Nov 2017 13:05:12 +0100
Subject: [PATCH 1/6] Use local timezone for TZ conversion in the FB system
 module

This adds a `convert_timezone` fileset parameter that, when enabled,
does two things:

* Uses the `add_locale` processor in the FB proespector config
* Uses `{{ beat.timezone }}` as the `timezone` parameter for the
  date processor in the Ingest Node pipeline. This parameter accepts
  templates starting with ES 6.1.

For the moment the `convert_timezone` flag is off by default, to keep
backwards compatibility and because it results in an error when used
with ES < 6.1.

Closes #3898.

For now this is only applied to the system module, but likely more
modules would benefit from this feature.
---
 filebeat/fileset/fileset.go                   | 34 ++++++++++++-------
 .../module/system/syslog/config/syslog.yml    |  4 +++
 .../module/system/syslog/ingest/pipeline.json |  5 +--
 filebeat/module/system/syslog/manifest.yml    |  2 ++
 testing/environments/latest.yml               |  6 ++--
 5 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
index 1f2fd260102..4cf3508604c 100644
--- a/filebeat/fileset/fileset.go
+++ b/filebeat/fileset/fileset.go
@@ -160,13 +160,13 @@ func (fs *Fileset) evaluateVars() (map[string]interface{}, error) {
 func resolveVariable(vars map[string]interface{}, value interface{}) (interface{}, error) {
 	switch v := value.(type) {
 	case string:
-		return applyTemplate(vars, v)
+		return applyTemplate(vars, v, false)
 	case []interface{}:
 		transformed := []interface{}{}
 		for _, val := range v {
 			s, ok := val.(string)
 			if ok {
-				transf, err := applyTemplate(vars, s)
+				transf, err := applyTemplate(vars, s, false)
 				if err != nil {
 					return nil, fmt.Errorf("array: %v", err)
 				}
@@ -180,9 +180,15 @@ func resolveVariable(vars map[string]interface{}, value interface{}) (interface{
 	return value, nil
 }
 
-// applyTemplate applies a Golang text/template
-func applyTemplate(vars map[string]interface{}, templateString string) (string, error) {
-	tpl, err := template.New("text").Parse(templateString)
+// applyTemplate applies a Golang text/template. If specialDelims is set to true,
+// the delimiters are set to `{%` and `%}` instead of `{{` and `}}`. These are easier to use
+// in pipeline definitions.
+func applyTemplate(vars map[string]interface{}, templateString string, specialDelims bool) (string, error) {
+	tpl := template.New("text")
+	if specialDelims {
+		tpl = tpl.Delims("{%", "%}")
+	}
+	tpl, err := tpl.Parse(templateString)
 	if err != nil {
 		return "", fmt.Errorf("Error parsing template %s: %v", templateString, err)
 	}
@@ -215,7 +221,7 @@ func (fs *Fileset) getBuiltinVars() (map[string]interface{}, error) {
 }
 
 func (fs *Fileset) getProspectorConfig() (*common.Config, error) {
-	path, err := applyTemplate(fs.vars, fs.manifest.Prospector)
+	path, err := applyTemplate(fs.vars, fs.manifest.Prospector, false)
 	if err != nil {
 		return nil, fmt.Errorf("Error expanding vars on the prospector path: %v", err)
 	}
@@ -224,7 +230,7 @@ func (fs *Fileset) getProspectorConfig() (*common.Config, error) {
 		return nil, fmt.Errorf("Error reading prospector file %s: %v", path, err)
 	}
 
-	yaml, err := applyTemplate(fs.vars, string(contents))
+	yaml, err := applyTemplate(fs.vars, string(contents), false)
 	if err != nil {
 		return nil, fmt.Errorf("Error interpreting the template of the prospector: %v", err)
 	}
@@ -269,7 +275,7 @@ func (fs *Fileset) getProspectorConfig() (*common.Config, error) {
 
 // getPipelineID returns the Ingest Node pipeline ID
 func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
-	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline)
+	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline, false)
 	if err != nil {
 		return "", fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)
 	}
@@ -278,18 +284,22 @@ func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
 }
 
 func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interface{}, err error) {
-	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline)
+	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline, false)
 	if err != nil {
 		return "", nil, fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)
 	}
 
-	f, err := os.Open(filepath.Join(fs.modulePath, fs.name, path))
+	strContents, err := ioutil.ReadFile(filepath.Join(fs.modulePath, fs.name, path))
 	if err != nil {
 		return "", nil, fmt.Errorf("Error reading pipeline file %s: %v", path, err)
 	}
 
-	dec := json.NewDecoder(f)
-	err = dec.Decode(&content)
+	jsonString, err := applyTemplate(fs.vars, string(strContents), true)
+	if err != nil {
+		return "", nil, fmt.Errorf("Error interpreting the template of the ingest pipeline: %v", err)
+	}
+
+	err = json.Unmarshal([]byte(jsonString), &content)
 	if err != nil {
 		return "", nil, fmt.Errorf("Error JSON decoding the pipeline file: %s: %v", path, err)
 	}
diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml
index 003b5d06b3e..3834faa1c9a 100644
--- a/filebeat/module/system/syslog/config/syslog.yml
+++ b/filebeat/module/system/syslog/config/syslog.yml
@@ -7,3 +7,7 @@ exclude_files: [".gz$"]
 multiline:
   pattern: "^\\s"
   match: after
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json
index 8555e697169..71f1cef0483 100644
--- a/filebeat/module/system/syslog/ingest/pipeline.json
+++ b/filebeat/module/system/syslog/ingest/pipeline.json
@@ -19,7 +19,7 @@
         "field": "message"
       }
     },
-		{
+    {
       "date": {
         "field": "system.syslog.timestamp",
         "target_field": "@timestamp",
@@ -27,9 +27,10 @@
 					"MMM  d HH:mm:ss",
 					"MMM dd HH:mm:ss"
         ],
+        {% if .convert_timezone %}"timezone": "{{ beat.timezone }}",{% end %}
         "ignore_failure": true
       }
-		}
+    }
 	],
   "on_failure" : [{
     "set" : {
diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml
index bd2ae28b40f..2252970af4a 100644
--- a/filebeat/module/system/syslog/manifest.yml
+++ b/filebeat/module/system/syslog/manifest.yml
@@ -8,6 +8,8 @@ var:
     os.darwin:
       - /var/log/system.log*
     os.windows: []
+  - name: convert_timezone
+    default: false
 
 ingest_pipeline: ingest/pipeline.json
 prospector: config/syslog.yml
diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml
index 0f614b57149..88ebb42c042 100644
--- a/testing/environments/latest.yml
+++ b/testing/environments/latest.yml
@@ -3,7 +3,7 @@
 version: '2.1'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0-rc2
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.0.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -18,13 +18,13 @@ services:
       context: docker/logstash
       dockerfile: Dockerfile
       args:
-        ELASTIC_VERSION: 6.0.0-rc2
+        ELASTIC_VERSION: 6.0.0
         DOWNLOAD_URL: https://artifacts.elastic.co/downloads
     environment:
       - ES_HOST=elasticsearch
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:6.0.0-rc2
+    image: docker.elastic.co/kibana/kibana:6.0.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 6

From 6a453911cd1f9d2d64fb6e8795143dd3a50fe4f0 Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Mon, 20 Nov 2017 15:43:50 +0100
Subject: [PATCH 2/6] Automatically turn off given options depending on the ES
 version.

---
 filebeat/fileset/fileset.go                | 51 +++++++++++++++++++++-
 filebeat/fileset/modules.go                |  2 +-
 filebeat/module/system/syslog/manifest.yml |  5 +++
 3 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
index 4cf3508604c..9a5b5dfdc12 100644
--- a/filebeat/fileset/fileset.go
+++ b/filebeat/fileset/fileset.go
@@ -17,6 +17,7 @@ import (
 	"text/template"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/logp"
 	mlimporter "github.com/elastic/beats/libbeat/ml-importer"
 )
 
@@ -155,6 +156,46 @@ func (fs *Fileset) evaluateVars() (map[string]interface{}, error) {
 	return vars, nil
 }
 
+// turnOffElasticsearchVars re-evaluates the variables that have `min_elasticsearch_version`
+// set.
+func (fs *Fileset) turnOffElasticsearchVars(vars map[string]interface{}, esVersion string) (map[string]interface{}, error) {
+
+	retVars := map[string]interface{}{}
+	for key, val := range vars {
+		retVars[key] = val
+	}
+
+	haveVersion, err := common.NewVersion(esVersion)
+	if err != nil {
+		return vars, fmt.Errorf("Error parsing version %s: %v", esVersion, err)
+	}
+
+	for _, vals := range fs.manifest.Vars {
+		var exists bool
+		name, exists := vals["name"].(string)
+		if !exists {
+			return nil, fmt.Errorf("Variable doesn't have a string 'name' key")
+		}
+
+		minESVersion, exists := vals["min_elasticsearch_version"].(map[string]interface{})
+		if exists {
+			minVersion, err := common.NewVersion(minESVersion["version"].(string))
+			if err != nil {
+				return vars, fmt.Errorf("Error parsing version %s: %v", minESVersion["version"].(string), err)
+			}
+
+			logp.Debug("fileset", "Comparing ES version %s with %s", haveVersion, minVersion)
+
+			if haveVersion.LessThan(minVersion) {
+				retVars[name] = minESVersion["value"]
+				logp.Info("Setting var %s to %v because Elasticsearch version is %s", name, minESVersion["value"], haveVersion)
+			}
+		}
+	}
+
+	return retVars, nil
+}
+
 // resolveVariable considers the value as a template so it can refer to built-in variables
 // as well as other variables defined before them.
 func resolveVariable(vars map[string]interface{}, value interface{}) (interface{}, error) {
@@ -283,7 +324,8 @@ func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
 	return formatPipelineID(fs.mcfg.Module, fs.name, path, beatVersion), nil
 }
 
-func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interface{}, err error) {
+func (fs *Fileset) GetPipeline(esVersion string) (pipelineID string, content map[string]interface{}, err error) {
+
 	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline, false)
 	if err != nil {
 		return "", nil, fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)
@@ -294,7 +336,12 @@ func (fs *Fileset) GetPipeline() (pipelineID string, content map[string]interfac
 		return "", nil, fmt.Errorf("Error reading pipeline file %s: %v", path, err)
 	}
 
-	jsonString, err := applyTemplate(fs.vars, string(strContents), true)
+	vars, err := fs.turnOffElasticsearchVars(fs.vars, esVersion)
+	if err != nil {
+		return "", nil, err
+	}
+
+	jsonString, err := applyTemplate(vars, string(strContents), true)
 	if err != nil {
 		return "", nil, fmt.Errorf("Error interpreting the template of the ingest pipeline: %v", err)
 	}
diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go
index d97fd77f548..550a304265a 100644
--- a/filebeat/fileset/modules.go
+++ b/filebeat/fileset/modules.go
@@ -278,7 +278,7 @@ func (reg *ModuleRegistry) LoadPipelines(esClient PipelineLoader) error {
 				}
 			}
 
-			pipelineID, content, err := fileset.GetPipeline()
+			pipelineID, content, err := fileset.GetPipeline(esClient.GetVersion())
 			if err != nil {
 				return fmt.Errorf("Error getting pipeline for fileset %s/%s: %v", module, name, err)
 			}
diff --git a/filebeat/module/system/syslog/manifest.yml b/filebeat/module/system/syslog/manifest.yml
index 2252970af4a..88cfe1bb578 100644
--- a/filebeat/module/system/syslog/manifest.yml
+++ b/filebeat/module/system/syslog/manifest.yml
@@ -10,6 +10,11 @@ var:
     os.windows: []
   - name: convert_timezone
     default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 prospector: config/syslog.yml

From 224d8dcb72d370aba009739f2d9cd3f2809be3b0 Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Mon, 20 Nov 2017 16:07:35 +0100
Subject: [PATCH 3/6] Added the convert_timezone flag to the auth fileset as
 well

---
 filebeat/fileset/fileset.go                      | 8 +++++++-
 filebeat/module/system/auth/config/auth.yml      | 4 ++++
 filebeat/module/system/auth/ingest/pipeline.json | 1 +
 filebeat/module/system/auth/manifest.yml         | 7 +++++++
 4 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
index 9a5b5dfdc12..de2593b0d3c 100644
--- a/filebeat/fileset/fileset.go
+++ b/filebeat/fileset/fileset.go
@@ -52,6 +52,11 @@ func New(
 	}, nil
 }
 
+// String returns the module and the name of the fileset.
+func (fs *Fileset) String() string {
+	return fs.mcfg.Module + "/" + fs.name
+}
+
 // Read reads the manifest file and evaluates the variables.
 func (fs *Fileset) Read(beatVersion string) error {
 	var err error
@@ -188,7 +193,7 @@ func (fs *Fileset) turnOffElasticsearchVars(vars map[string]interface{}, esVersi
 
 			if haveVersion.LessThan(minVersion) {
 				retVars[name] = minESVersion["value"]
-				logp.Info("Setting var %s to %v because Elasticsearch version is %s", name, minESVersion["value"], haveVersion)
+				logp.Info("Setting var %s (%s) to %v because Elasticsearch version is %s", name, fs, minESVersion["value"], haveVersion)
 			}
 		}
 	}
@@ -324,6 +329,7 @@ func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
 	return formatPipelineID(fs.mcfg.Module, fs.name, path, beatVersion), nil
 }
 
+// GetPipeline returns the JSON content of the Ingest Node pipeline that parses the logs.
 func (fs *Fileset) GetPipeline(esVersion string) (pipelineID string, content map[string]interface{}, err error) {
 
 	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline, false)
diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml
index 003b5d06b3e..3834faa1c9a 100644
--- a/filebeat/module/system/auth/config/auth.yml
+++ b/filebeat/module/system/auth/config/auth.yml
@@ -7,3 +7,7 @@ exclude_files: [".gz$"]
 multiline:
   pattern: "^\\s"
   match: after
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json
index a305a5b1e8c..86bda2e0d14 100644
--- a/filebeat/module/system/auth/ingest/pipeline.json
+++ b/filebeat/module/system/auth/ingest/pipeline.json
@@ -32,6 +32,7 @@
 					"MMM  d HH:mm:ss",
 					"MMM dd HH:mm:ss"
         ],
+        {% if .convert_timezone %}"timezone": "{{ beat.timezone }}",{% end %}
         "ignore_failure": true
       }
     },
diff --git a/filebeat/module/system/auth/manifest.yml b/filebeat/module/system/auth/manifest.yml
index bb391be05b1..c64de896eb0 100644
--- a/filebeat/module/system/auth/manifest.yml
+++ b/filebeat/module/system/auth/manifest.yml
@@ -10,6 +10,13 @@ var:
       # ssh logs to files
       - /var/log/secure.log*
     os.windows: []
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 prospector: config/auth.yml

From fcbc8c754b6fcc5b98737938ea5987b54902e391 Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Mon, 20 Nov 2017 21:10:44 +0100
Subject: [PATCH 4/6] Added tests

---
 filebeat/fileset/fileset_test.go | 46 +++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/filebeat/fileset/fileset_test.go b/filebeat/fileset/fileset_test.go
index e88588aa746..09010642358 100644
--- a/filebeat/fileset/fileset_test.go
+++ b/filebeat/fileset/fileset_test.go
@@ -3,12 +3,15 @@
 package fileset
 
 import (
+	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/logp"
 )
 
 func getModuleForTesting(t *testing.T, module, fileset string) *Fileset {
@@ -193,9 +196,50 @@ func TestGetPipelineNginx(t *testing.T) {
 	fs := getModuleForTesting(t, "nginx", "access")
 	assert.NoError(t, fs.Read("5.2.0"))
 
-	pipelineID, content, err := fs.GetPipeline()
+	pipelineID, content, err := fs.GetPipeline("5.2.0")
 	assert.NoError(t, err)
 	assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipelineID)
 	assert.Contains(t, content, "description")
 	assert.Contains(t, content, "processors")
 }
+
+func TestGetPipelineConvertTS(t *testing.T) {
+	if testing.Verbose() {
+		logp.LogInit(logp.LOG_DEBUG, "", false, true, []string{"fileset", "modules"})
+	}
+
+	// load system/syslog
+	modulesPath, err := filepath.Abs("../module")
+	assert.NoError(t, err)
+	fs, err := New(modulesPath, "syslog", &ModuleConfig{Module: "system"}, &FilesetConfig{
+		Var: map[string]interface{}{
+			"convert_timezone": true,
+		},
+	})
+	assert.NoError(t, err)
+	assert.NoError(t, fs.Read("6.1.0"))
+
+	// ES 6.0.0 should not have beat.timezone referenced
+	pipelineID, content, err := fs.GetPipeline("6.0.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "filebeat-6.1.0-system-syslog-pipeline", pipelineID)
+	marshaled, err := json.Marshal(content)
+	assert.NoError(t, err)
+	assert.NotContains(t, string(marshaled), "beat.timezone")
+
+	// ES 6.1.0 should have beat.timezone referenced
+	pipelineID, content, err = fs.GetPipeline("6.1.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "filebeat-6.1.0-system-syslog-pipeline", pipelineID)
+	marshaled, err = json.Marshal(content)
+	assert.NoError(t, err)
+	assert.Contains(t, string(marshaled), "beat.timezone")
+
+	// ES 6.2.0 should have beat.timezone referenced
+	pipelineID, content, err = fs.GetPipeline("6.2.0")
+	assert.NoError(t, err)
+	assert.Equal(t, "filebeat-6.1.0-system-syslog-pipeline", pipelineID)
+	marshaled, err = json.Marshal(content)
+	assert.NoError(t, err)
+	assert.Contains(t, string(marshaled), "beat.timezone")
+}

From aa397f97d1f5a065fb9722291dad608837c3e1cd Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Tue, 21 Nov 2017 00:55:29 +0100
Subject: [PATCH 5/6] Docs & changelog

---
 CHANGELOG.asciidoc                                 |  1 +
 .../docs/include/var-convert-timezone.asciidoc     |  8 ++++++++
 filebeat/docs/modules/system.asciidoc              | 14 ++++++++++++--
 filebeat/filebeat.reference.yml                    |  6 ++++++
 filebeat/module/system/_meta/config.reference.yml  |  6 ++++++
 filebeat/module/system/_meta/config.yml            |  6 ++++++
 filebeat/module/system/_meta/docs.asciidoc         | 14 ++++++++++++--
 filebeat/modules.d/system.yml.disabled             |  6 ++++++
 8 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100644 filebeat/docs/include/var-convert-timezone.asciidoc

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 813df6f2ce4..5b80af334d7 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add Kubernetes manifests to deploy Filebeat. {pull}5349[5349]
 - Add experimental Docker `json-file` prospector . {pull}5402[5402]
 - Add experimental Docker autodiscover functionality. {pull}5245[5245]
+- Add option to convert the timestamps to UTC in the system module. {pull}5647[5647]
 
 *Heartbeat*
 
diff --git a/filebeat/docs/include/var-convert-timezone.asciidoc b/filebeat/docs/include/var-convert-timezone.asciidoc
new file mode 100644
index 00000000000..52c71d84406
--- /dev/null
+++ b/filebeat/docs/include/var-convert-timezone.asciidoc
@@ -0,0 +1,8 @@
+*`var.convert_timezone`*::
+
+If this option is enabled, Filebeat reads the local timezone and uses it at log
+parsing time to convert the timestamp to UTC. The local timezone is also added
+in each event in a dedicated field (`beat.timezone`). The conversion is only
+possible in Elasticsearch >= 6.1. If the Elasticsearch version is less than 6.1,
+the `beat.timezone` field is added, but the conversion to UTC is not made.  The
+default is `false`.
diff --git a/filebeat/docs/modules/system.asciidoc b/filebeat/docs/modules/system.asciidoc
index b9955446ce8..d809b41b679 100644
--- a/filebeat/docs/modules/system.asciidoc
+++ b/filebeat/docs/modules/system.asciidoc
@@ -33,7 +33,7 @@ image::./images/kibana-system.png[]
 include::../include/configuring-intro.asciidoc[]
 
 The following example shows how to set paths in the +modules.d/{modulename}.yml+
-file to override the default paths for the syslog and authorization logs: 
+file to override the default paths for the syslog and authorization logs:
 
 ["source","yaml",subs="attributes"]
 -----
@@ -55,7 +55,7 @@ To specify the same settings at the command line, you use:
 -----
 
 
-The command in the example assumes that you have already enabled the +{modulename}+ module. 
+The command in the example assumes that you have already enabled the +{modulename}+ module.
 
 //set the fileset name used in the included example
 :fileset_ex: syslog
@@ -68,6 +68,16 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/var-convert-timezone.asciidoc[]
+
+[float]
+==== `auth` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/var-convert-timezone.asciidoc[]
+
+
 
 
 [float]
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
index 293d834a79c..c10cac71b0e 100644
--- a/filebeat/filebeat.reference.yml
+++ b/filebeat/filebeat.reference.yml
@@ -21,6 +21,9 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
     # Prospector configuration (advanced). Any prospector configuration option
     # can be added under this section.
     #prospector:
@@ -33,6 +36,9 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
     # Prospector configuration (advanced). Any prospector configuration option
     # can be added under this section.
     #prospector:
diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml
index 266677dc118..dc222d2a44c 100644
--- a/filebeat/module/system/_meta/config.reference.yml
+++ b/filebeat/module/system/_meta/config.reference.yml
@@ -7,6 +7,9 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
     # Prospector configuration (advanced). Any prospector configuration option
     # can be added under this section.
     #prospector:
@@ -19,6 +22,9 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
     # Prospector configuration (advanced). Any prospector configuration option
     # can be added under this section.
     #prospector:
diff --git a/filebeat/module/system/_meta/config.yml b/filebeat/module/system/_meta/config.yml
index f76dd905b4d..4513d921894 100644
--- a/filebeat/module/system/_meta/config.yml
+++ b/filebeat/module/system/_meta/config.yml
@@ -7,6 +7,9 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
   # Authorization logs
   auth:
     enabled: true
@@ -14,3 +17,6 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
diff --git a/filebeat/module/system/_meta/docs.asciidoc b/filebeat/module/system/_meta/docs.asciidoc
index caae477ed92..3d98efd8374 100644
--- a/filebeat/module/system/_meta/docs.asciidoc
+++ b/filebeat/module/system/_meta/docs.asciidoc
@@ -28,7 +28,7 @@ image::./images/kibana-system.png[]
 include::../include/configuring-intro.asciidoc[]
 
 The following example shows how to set paths in the +modules.d/{modulename}.yml+
-file to override the default paths for the syslog and authorization logs: 
+file to override the default paths for the syslog and authorization logs:
 
 ["source","yaml",subs="attributes"]
 -----
@@ -50,7 +50,7 @@ To specify the same settings at the command line, you use:
 -----
 
 
-The command in the example assumes that you have already enabled the +{modulename}+ module. 
+The command in the example assumes that you have already enabled the +{modulename}+ module.
 
 //set the fileset name used in the included example
 :fileset_ex: syslog
@@ -63,3 +63,13 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/var-convert-timezone.asciidoc[]
+
+[float]
+==== `auth` fileset settings
+
+include::../include/var-paths.asciidoc[]
+
+include::../include/var-convert-timezone.asciidoc[]
+
+
diff --git a/filebeat/modules.d/system.yml.disabled b/filebeat/modules.d/system.yml.disabled
index f76dd905b4d..4513d921894 100644
--- a/filebeat/modules.d/system.yml.disabled
+++ b/filebeat/modules.d/system.yml.disabled
@@ -7,6 +7,9 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false
+
   # Authorization logs
   auth:
     enabled: true
@@ -14,3 +17,6 @@
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #convert_timezone: false

From 0221d62dea2874896120e2200bf1ce08bdff9e5f Mon Sep 17 00:00:00 2001
From: Tudor Golubenco <tudor@elastic.co>
Date: Tue, 21 Nov 2017 09:48:09 +0100
Subject: [PATCH 6/6] Addressed comments

---
 filebeat/fileset/fileset.go | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go
index de2593b0d3c..17d2d6fa331 100644
--- a/filebeat/fileset/fileset.go
+++ b/filebeat/fileset/fileset.go
@@ -164,7 +164,6 @@ func (fs *Fileset) evaluateVars() (map[string]interface{}, error) {
 // turnOffElasticsearchVars re-evaluates the variables that have `min_elasticsearch_version`
 // set.
 func (fs *Fileset) turnOffElasticsearchVars(vars map[string]interface{}, esVersion string) (map[string]interface{}, error) {
-
 	retVars := map[string]interface{}{}
 	for key, val := range vars {
 		retVars[key] = val
@@ -176,20 +175,20 @@ func (fs *Fileset) turnOffElasticsearchVars(vars map[string]interface{}, esVersi
 	}
 
 	for _, vals := range fs.manifest.Vars {
-		var exists bool
-		name, exists := vals["name"].(string)
-		if !exists {
+		var ok bool
+		name, ok := vals["name"].(string)
+		if !ok {
 			return nil, fmt.Errorf("Variable doesn't have a string 'name' key")
 		}
 
-		minESVersion, exists := vals["min_elasticsearch_version"].(map[string]interface{})
-		if exists {
+		minESVersion, ok := vals["min_elasticsearch_version"].(map[string]interface{})
+		if ok {
 			minVersion, err := common.NewVersion(minESVersion["version"].(string))
 			if err != nil {
 				return vars, fmt.Errorf("Error parsing version %s: %v", minESVersion["version"].(string), err)
 			}
 
-			logp.Debug("fileset", "Comparing ES version %s with %s", haveVersion, minVersion)
+			logp.Debug("fileset", "Comparing ES version %s with requirement of %s", haveVersion, minVersion)
 
 			if haveVersion.LessThan(minVersion) {
 				retVars[name] = minESVersion["value"]
@@ -331,7 +330,6 @@ func (fs *Fileset) getPipelineID(beatVersion string) (string, error) {
 
 // GetPipeline returns the JSON content of the Ingest Node pipeline that parses the logs.
 func (fs *Fileset) GetPipeline(esVersion string) (pipelineID string, content map[string]interface{}, err error) {
-
 	path, err := applyTemplate(fs.vars, fs.manifest.IngestPipeline, false)
 	if err != nil {
 		return "", nil, fmt.Errorf("Error expanding vars on the ingest pipeline path: %v", err)