Remove _all field from template for ES >= 6.0 (#3778)

ES 6.0 deprecates `_all`, see elastic/elasticsearch#22144.

`*.template-es6x.json` has been added in case Elasticsearch 6.X is
detected

CHANGELOG.asciidoc

@@ -64,6 +64,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add the option to load the sample dashboards during the Beat startup phase. {pull}3506[3506]
 - Disabled date detection in Elasticsearch index templates. Date fields must be explicitly defined in index templates. {pull}3528[3528]
 - Using environment variables in the configuration file is now GA, instead of experimental. {pull}3525[3525]
+- Update index mappings to support future Elasticsearch 6.X. {pull}3778[3778]
 
 *Filebeat*
 

dev-tools/packer/platforms/binary/run.sh.j2

@@ -18,6 +18,7 @@ cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.be
 cp {{.beat_name}}-linux.full.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.full.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
 cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
+cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}

dev-tools/packer/platforms/centos/run.sh.j2

@@ -38,6 +38,7 @@ fpm --force -s dir -t rpm \
         {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \
         {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
         {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
+        {{.beat_name}}.template-es6x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es6x.json \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \
         import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards

dev-tools/packer/platforms/darwin/run.sh.j2

@@ -18,6 +18,7 @@ cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_na
 cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.full.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
 cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64

dev-tools/packer/platforms/debian/run.sh.j2

@@ -35,6 +35,7 @@ fpm --force -s dir -t deb \
         {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \
         {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
         {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
+        {{.beat_name}}.template-es6x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es6x.json \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \
         import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards

dev-tools/packer/platforms/windows/run.sh.j2

@@ -19,6 +19,7 @@ cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.be
 cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.full.yml
 cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp {{.beat_name}}.template-es6x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 

dev-tools/packer/xgo-scripts/before_build.sh

@@ -15,6 +15,7 @@ PREFIX=/build
 # Copy template
 cp $BEAT_NAME.template.json $PREFIX/$BEAT_NAME.template.json
 cp $BEAT_NAME.template-es2x.json $PREFIX/$BEAT_NAME.template-es2x.json
+cp $BEAT_NAME.template-es6x.json $PREFIX/$BEAT_NAME.template-es6x.json
 
 # linux
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml

filebeat/filebeat.full.yml

@@ -510,6 +510,14 @@ output.elasticsearch:
   # Path to the Elasticsearch 2.x version of the template file.
   #template.versions.2x.path: "${path.config}/filebeat.template-es2x.json"
 
+  # If set to true, filebeat checks the Elasticsearch version at connect time, and if it
+  # is 6.x, it loads the file specified by the template.versions.6x.path setting. The
+  # default is true.
+  #template.versions.6x.enabled: true
+
+  # Path to the Elasticsearch 6.x version of the template file.
+  #template.versions.6x.path: "${path.config}/filebeat.template-es6x.json"
+
   # Use SSL settings for HTTPS. Default is true.
   #ssl.enabled: true