Dynamically generate template on startup (#3681)

* Dynamically generate template on startup

So far the template was loaded from the `beatname.template.json` files. This now uses the `fields.yml` file to dynamically generate the template based on the ES version connected to. The name of the template is set dynamically based on `template.name`. By default it has the value {beatname}.

* [x] Remove generated templates
* [x] Package fields.yml
* [x] Clean up config options as 2.x option is not needed anymore
* [x] Update docs
* [x] Update and test packaging

Further changes:

* Remove tests in libbeat to load all templates. This requires global knowledge of the other beats in libbeat which is not good.
* Create new makefile target to generate template for testing purpose.

Questions:
* Config option is called `template.path` which points to `fields.yml`. Is the name still ok?
* What is our recommendation on loading the template manually? Generate first with `index_template` binary and then load manually?
  * Do we ship in index_template binary similar to import_dashboards? Can we integrate it into beats directly? -> setup
* Should template config be moved to the top level (like `dashboard`)

Part of #3654

* rename template.path to template.fields

dev-tools/packer/platforms/binary/run.sh.j2

@@ -16,8 +16,7 @@ install -D -m 755 import_dashboards-linux-{{.arch}} /{{.beat_name}}-${VERSION}-l
 cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}
 cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-linux.full.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.full.yml
-cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
-cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
+cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}

dev-tools/packer/platforms/centos/run.sh.j2

@@ -36,8 +36,7 @@ fpm --force -s dir -t rpm \
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
         {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \
-        {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
-        {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
+        fields.yml=/etc/{{.beat_name}}/fields.yml \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \
         import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards

dev-tools/packer/platforms/darwin/run.sh.j2

@@ -16,8 +16,7 @@ install -D -m 755 import_dashboards-darwin-{{.arch}} /{{.beat_name}}-${VERSION}-
 cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}
 cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml
 cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.full.yml
-cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
-cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/
+cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/
 
 mkdir -p upload
 tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64

dev-tools/packer/platforms/debian/run.sh.j2

@@ -33,8 +33,7 @@ fpm --force -s dir -t deb \
         {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \
         {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \
         {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \
-        {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \
-        {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \
+        fields.yml=/etc/{{.beat_name}}/fields.yml \
         ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \
         god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \
         import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards

dev-tools/packer/platforms/windows/run.sh.j2

@@ -17,8 +17,7 @@ cp {{.beat_name}}-windows-{{.arch}}.exe /{{.beat_name}}-${VERSION}-windows-{{.wi
 unix2dos {{.beat_name}}-win.yml
 cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml
 cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.full.yml
-cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
-cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
+cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/
 

dev-tools/packer/xgo-scripts/before_build.sh

@@ -12,9 +12,8 @@ cd $GOPATH/src/$BEAT_PATH
 
 PREFIX=/build
 
-# Copy template
-cp $BEAT_NAME.template.json $PREFIX/$BEAT_NAME.template.json
-cp $BEAT_NAME.template-es2x.json $PREFIX/$BEAT_NAME.template-es2x.json
+# Copy fields.yml
+cp fields.yml $PREFIX/fields.yml
 
 # linux
 cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml

filebeat/filebeat.full.yml

@@ -498,20 +498,12 @@ output.elasticsearch:
   # so the final name is filebeat-%{[beat.version]}.
   #template.name: "filebeat"
 
-  # Path to template file
-  #template.path: "${path.config}/filebeat.template.json"
+  # Path to fields.yml file to generate the template
+  #template.fields: "${path.config}/fields.yml"
 
   # Overwrite existing template
   #template.overwrite: false
 
-  # If set to true, filebeat checks the Elasticsearch version at connect time, and if it
-  # is 2.x, it loads the file specified by the template.versions.2x.path setting. The
-  # default is true.
-  #template.versions.2x.enabled: true
-
-  # Path to the Elasticsearch 2.x version of the template file.
-  #template.versions.2x.path: "${path.config}/filebeat.template-es2x.json"
-
   # Use SSL settings for HTTPS. Default is true.
   #ssl.enabled: true
 

generator/beat/{beat}/README.md

@@ -70,8 +70,7 @@ The test coverage is reported in the folder `./build/coverage/`
 ### Update
 
 Each beat has a template for the mapping in elasticsearch and a documentation for the fields
-which is automatically generated based on `etc/fields.yml`.
-To generate etc/{beat}.template.json and etc/{beat}.asciidoc
+which is automatically generated based on `fields.yml` by running the following command.
 
 ```
 make update

heartbeat/README.md

@@ -35,8 +35,7 @@ To run Heartbeat with debugging output enabled, run:
 ### Update
 
 Each beat has a template for the mapping in elasticsearch and a documentation for the fields
-which is automatically generated based on `etc/fields.yml`.
-To generate etc/heartbeat.template.json and etc/heartbeat.asciidoc
+which is automatically generated based on `fields.yml`.
 
 ```
 make update

heartbeat/heartbeat.full.yml

@@ -346,20 +346,12 @@ output.elasticsearch:
   # so the final name is heartbeat-%{[beat.version]}.
   #template.name: "heartbeat"
 
-  # Path to template file
-  #template.path: "${path.config}/heartbeat.template.json"
+  # Path to fields.yml file to generate the template
+  #template.fields: "${path.config}/fields.yml"
 
   # Overwrite existing template
   #template.overwrite: false
 
-  # If set to true, heartbeat checks the Elasticsearch version at connect time, and if it
-  # is 2.x, it loads the file specified by the template.versions.2x.path setting. The
-  # default is true.
-  #template.versions.2x.enabled: true
-
-  # Path to the Elasticsearch 2.x version of the template file.
-  #template.versions.2x.path: "${path.config}/heartbeat.template-es2x.json"
-
   # Use SSL settings for HTTPS. Default is true.
   #ssl.enabled: true
 

libbeat/.gitignore

@@ -25,8 +25,6 @@ _testmain.go
 .idea
 .jenkins
 
-/libbeat.template-es2x.json
 /libbeat.yml
 /libbeat.full.yml
-/libbeat.template.json
 /docs/fields.asciidoc

libbeat/Makefile

@@ -14,3 +14,6 @@ fields:
 # Collects all dependencies and then calls update
 .PHONY: collect
 collect: fields
+
+clean::
+	rm -rf libbeat.full.yml libbeat.yml

libbeat/_meta/config.full.yml

@@ -148,20 +148,12 @@ output.elasticsearch:
   # so the final name is beatname-%{[beat.version]}.
   #template.name: "beatname"
 
-  # Path to template file
-  #template.path: "${path.config}/beatname.template.json"
+  # Path to fields.yml file to generate the template
+  #template.fields: "${path.config}/fields.yml"
 
   # Overwrite existing template
   #template.overwrite: false
 
-  # If set to true, beatname checks the Elasticsearch version at connect time, and if it
-  # is 2.x, it loads the file specified by the template.versions.2x.path setting. The
-  # default is true.
-  #template.versions.2x.enabled: true
-
-  # Path to the Elasticsearch 2.x version of the template file.
-  #template.versions.2x.path: "${path.config}/beatname.template-es2x.json"
-
   # Use SSL settings for HTTPS. Default is true.
   #ssl.enabled: true
 

libbeat/docs/outputconfig.asciidoc

@@ -23,7 +23,7 @@ Example configuration:
 output.elasticsearch:
   hosts: ["http://localhost:9200"]
   template.enabled: true
-  template.path: "{beatname_lc}.template.json"
+  template.fields: "fields.yml"
   template.overwrite: false
   index: "{beatname_lc}"
   ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
@@ -253,7 +253,7 @@ you must <<load-template-manually,load the template manually>>.
 
 *`name`*:: The name of the template. The default is +{beatname_lc}+.
 
-*`path`*:: The path to the template file. The default is +{beatname_lc}.template.json+. If a relative
+*`path`*:: The path to the template file. The default is +fields.yml+. If a relative
 path is set, it is considered relative to the config path. See the <<directory-layout>> section for
 details.
 
@@ -267,7 +267,7 @@ For example:
 output.elasticsearch:
   hosts: ["localhost:9200"]
   template.name: "{beatname_lc}"
-  template.path: "{beatname_lc}.template.json"
+  template.fields: "fields.yml"
   template.overwrite: false
 ----------------------------------------------------------------------
 
@@ -289,9 +289,8 @@ For example:
 ----------------------------------------------------------------------
 output.elasticsearch:
   hosts: ["localhost:9200"]
-  template.path: "{beatname_lc}.template.json"
+  template.fields: "{beatname_lc}.template.json"
   template.overwrite: false
-  template.versions.2x.path: "{beatname_lc}.template-es2x.json
 ----------------------------------------------------------------------
 
 ===== max_retries

libbeat/docs/shared-template-load.asciidoc

@@ -17,16 +17,16 @@
 
 
 In Elasticsearch, {elasticsearch}/indices-templates.html[index
-templates] are used to define settings and mappings that determine how fields should be analyzed. 
+templates] are used to define settings and mappings that determine how fields should be analyzed.
 
 The recommended index template file for {beatname_uc} is installed by the {beatname_uc} packages. If you accept
-the default configuration for template loading in the +{beatname_lc}.yml+ config file, 
+the default configuration for template loading in the +{beatname_lc}.yml+ config file,
 {beatname_uc} loads the template automatically after successfully connecting to Elasticsearch. If the template
 already exists, it's not overwritten unless you configure {beatname_uc} to do so.
 
-If you want to disable automatic template loading, or you want to load your own template, 
-you can change the settings for template loading in the {beatname_uc} configuration file. If you 
-choose to disable automatic template loading, you need to load the template manually. 
+If you want to disable automatic template loading, or you want to load your own template,
+you can change the settings for template loading in the {beatname_uc} configuration file. If you
+choose to disable automatic template loading, you need to load the template manually.
 For more information, see:
 
 * <<load-template-auto>> - supported for Elasticsearch output only
@@ -35,16 +35,16 @@ For more information, see:
 [[load-template-auto]]
 ==== Configuring Template Loading
 
-By default, {beatname_uc} automatically loads the recommended template file, +{beatname_lc}.template.json+,
+By default, {beatname_uc} automatically loads the recommended template file, +fields.yml+,
 if Elasticsearch output is enabled. You can configure {beatname_lc} to load a different template
-by adjusting the `template.name` and `template.path` options in +{beatname_lc}.yml+ file:
+by adjusting the `template.name` and `template.fields` options in +{beatname_lc}.yml+ file:
 
 ["source","yaml",subs="attributes,callouts"]
 ----------------------------------------------------------------------
 output.elasticsearch:
   hosts: ["localhost:9200"]
   template.name: "{beatname_lc}"
-  template.path: "{beatname_lc}.template.json"
+  template.fields: "fields.yml"
   template.overwrite: false
 ----------------------------------------------------------------------
 

libbeat/outputs/elasticsearch/client.go

@@ -549,7 +549,6 @@ func itemStatusInner(reader *jsonReader) (int, []byte, error) {
 // PublishEvent publishes an event.
 func (client *Client) PublishEvent(data outputs.Data) error {
 	// insert the events one by one
-
 	event := data.Event
 	index := getIndex(event, client.index)
 
@@ -596,6 +595,7 @@ func (client *Client) PublishEvent(data outputs.Data) error {
 // then use CheckTemplate prior to calling this method.
 func (client *Client) LoadTemplate(templateName string, template map[string]interface{}) error {
 
+	logp.Info("load template: %s", templateName)
 	path := "/_template/" + templateName
 	body, err := client.LoadJSON(path, template)
 	if err != nil {