From bfa312871f60949dda190f2adf997896fdc255f8 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 4 Apr 2017 10:50:10 +0200 Subject: [PATCH] Dynamically generate template on startup (#3681) * Dynamically generate template on startup So far the template was loaded from the `beatname.template.json` files. This now uses the `fields.yml` file to dynamically generate the template based on the ES version connected to. The name of the template is set dynamically based on `template.name`. By default it has the value {beatname}. * [x] Remove generated templates * [x] Package fields.yml * [x] Clean up config options as 2.x option is not needed anymore * [x] Update docs * [x] Update and test packaging Further changes: * Remove tests in libbeat to load all templates. This requires global knowledge of the other beats in libbeat which is not good. * Create new makefile target to generate template for testing purpose. Questions: * Config option is called `template.path` which points to `fields.yml`. Is the name still ok? * What is our recommendation on loading the template manually? Generate first with `index_template` binary and then load manually? * Do we ship in index_template binary similar to import_dashboards? Can we integrate it into beats directly? -> setup * Should template config be moved to the top level (like `dashboard`) Part of https://github.com/elastic/beats/issues/3654 * rename template.path to template.fields --- dev-tools/packer/platforms/binary/run.sh.j2 | 3 +- dev-tools/packer/platforms/centos/run.sh.j2 | 3 +- dev-tools/packer/platforms/darwin/run.sh.j2 | 3 +- dev-tools/packer/platforms/debian/run.sh.j2 | 3 +- dev-tools/packer/platforms/windows/run.sh.j2 | 3 +- dev-tools/packer/xgo-scripts/before_build.sh | 5 +- filebeat/filebeat.full.yml | 12 +-- generator/beat/{beat}/README.md | 3 +- heartbeat/README.md | 3 +- heartbeat/heartbeat.full.yml | 12 +-- libbeat/.gitignore | 2 - libbeat/Makefile | 3 + libbeat/_meta/config.full.yml | 12 +-- libbeat/docs/outputconfig.asciidoc | 9 +- libbeat/docs/shared-template-load.asciidoc | 16 ++-- libbeat/outputs/elasticsearch/client.go | 2 +- .../elasticsearch/client_integration_test.go | 77 +++++++-------- libbeat/outputs/elasticsearch/config.go | 13 ++- libbeat/outputs/elasticsearch/output.go | 96 +++++-------------- libbeat/paths/paths.go | 1 + libbeat/scripts/Makefile | 9 +- metricbeat/metricbeat.full.yml | 12 +-- packetbeat/packetbeat.full.yml | 12 +-- winlogbeat/winlogbeat.full.yml | 12 +-- 24 files changed, 106 insertions(+), 220 deletions(-) diff --git a/dev-tools/packer/platforms/binary/run.sh.j2 b/dev-tools/packer/platforms/binary/run.sh.j2 index 9ae01a102b6..428221e8148 100644 --- a/dev-tools/packer/platforms/binary/run.sh.j2 +++ b/dev-tools/packer/platforms/binary/run.sh.j2 @@ -16,8 +16,7 @@ install -D -m 755 import_dashboards-linux-{{.arch}} /{{.beat_name}}-${VERSION}-l cp {{.beat_name}}-linux-{{.arch}} /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}} cp {{.beat_name}}-linux.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.yml cp {{.beat_name}}-linux.full.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/{{.beat_name}}.full.yml -cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ -cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ +cp fields.yml /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}/ mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-linux-{{.bin_arch}}.tar.gz /{{.beat_name}}-${VERSION}-linux-{{.bin_arch}} diff --git a/dev-tools/packer/platforms/centos/run.sh.j2 b/dev-tools/packer/platforms/centos/run.sh.j2 index d5df40c6e11..4f52f34c336 100644 --- a/dev-tools/packer/platforms/centos/run.sh.j2 +++ b/dev-tools/packer/platforms/centos/run.sh.j2 @@ -36,8 +36,7 @@ fpm --force -s dir -t rpm \ {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \ {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \ {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \ - {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \ - {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \ + fields.yml=/etc/{{.beat_name}}/fields.yml \ ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \ god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \ import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards diff --git a/dev-tools/packer/platforms/darwin/run.sh.j2 b/dev-tools/packer/platforms/darwin/run.sh.j2 index f246aeecafd..a2e8c65afbe 100644 --- a/dev-tools/packer/platforms/darwin/run.sh.j2 +++ b/dev-tools/packer/platforms/darwin/run.sh.j2 @@ -16,8 +16,7 @@ install -D -m 755 import_dashboards-darwin-{{.arch}} /{{.beat_name}}-${VERSION}- cp {{.beat_name}}-darwin-amd64 /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}} cp {{.beat_name}}-darwin.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.yml cp {{.beat_name}}-darwin.full.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/{{.beat_name}}.full.yml -cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-darwin-x86_64/ -cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-darwin-x86_64/ +cp fields.yml /{{.beat_name}}-${VERSION}-darwin-x86_64/ mkdir -p upload tar czvf upload/{{.beat_name}}-${VERSION}-darwin-x86_64.tar.gz /{{.beat_name}}-${VERSION}-darwin-x86_64 diff --git a/dev-tools/packer/platforms/debian/run.sh.j2 b/dev-tools/packer/platforms/debian/run.sh.j2 index 95ff01792be..e6e3853fffb 100644 --- a/dev-tools/packer/platforms/debian/run.sh.j2 +++ b/dev-tools/packer/platforms/debian/run.sh.j2 @@ -33,8 +33,7 @@ fpm --force -s dir -t deb \ {{.beat_name}}-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}} \ {{.beat_name}}-linux.yml=/etc/{{.beat_name}}/{{.beat_name}}.yml \ {{.beat_name}}-linux.full.yml=/etc/{{.beat_name}}/{{.beat_name}}.full.yml \ - {{.beat_name}}.template.json=/etc/{{.beat_name}}/{{.beat_name}}.template.json \ - {{.beat_name}}.template-es2x.json=/etc/{{.beat_name}}/{{.beat_name}}.template-es2x.json \ + fields.yml=/etc/{{.beat_name}}/fields.yml \ ${RUNID}.service=/lib/systemd/system/{{.beat_name}}.service \ god-linux-{{.arch}}=/usr/share/{{.beat_name}}/bin/{{.beat_name}}-god \ import_dashboards-linux-{{.arch}}=/usr/share/{{.beat_name}}/scripts/import_dashboards diff --git a/dev-tools/packer/platforms/windows/run.sh.j2 b/dev-tools/packer/platforms/windows/run.sh.j2 index d35643cd1d3..1fa180db5ad 100644 --- a/dev-tools/packer/platforms/windows/run.sh.j2 +++ b/dev-tools/packer/platforms/windows/run.sh.j2 @@ -17,8 +17,7 @@ cp {{.beat_name}}-windows-{{.arch}}.exe /{{.beat_name}}-${VERSION}-windows-{{.wi unix2dos {{.beat_name}}-win.yml cp {{.beat_name}}-win.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.yml cp {{.beat_name}}-win.full.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/{{.beat_name}}.full.yml -cp {{.beat_name}}.template.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ -cp {{.beat_name}}.template-es2x.json /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ +cp fields.yml /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp install-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ cp uninstall-service-{{.beat_name}}.ps1 /{{.beat_name}}-${VERSION}-windows-{{.win_arch}}/ diff --git a/dev-tools/packer/xgo-scripts/before_build.sh b/dev-tools/packer/xgo-scripts/before_build.sh index 0c839caa433..c1b985067ee 100755 --- a/dev-tools/packer/xgo-scripts/before_build.sh +++ b/dev-tools/packer/xgo-scripts/before_build.sh @@ -12,9 +12,8 @@ cd $GOPATH/src/$BEAT_PATH PREFIX=/build -# Copy template -cp $BEAT_NAME.template.json $PREFIX/$BEAT_NAME.template.json -cp $BEAT_NAME.template-es2x.json $PREFIX/$BEAT_NAME.template-es2x.json +# Copy fields.yml +cp fields.yml $PREFIX/fields.yml # linux cp $BEAT_NAME.yml $PREFIX/$BEAT_NAME-linux.yml diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 3a13b9fc34d..f3caf49222a 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -498,20 +498,12 @@ output.elasticsearch: # so the final name is filebeat-%{[beat.version]}. #template.name: "filebeat" - # Path to template file - #template.path: "${path.config}/filebeat.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, filebeat checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/filebeat.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/generator/beat/{beat}/README.md b/generator/beat/{beat}/README.md index 4441008385e..bce2a9c94fc 100644 --- a/generator/beat/{beat}/README.md +++ b/generator/beat/{beat}/README.md @@ -70,8 +70,7 @@ The test coverage is reported in the folder `./build/coverage/` ### Update Each beat has a template for the mapping in elasticsearch and a documentation for the fields -which is automatically generated based on `etc/fields.yml`. -To generate etc/{beat}.template.json and etc/{beat}.asciidoc +which is automatically generated based on `fields.yml` by running the following command. ``` make update diff --git a/heartbeat/README.md b/heartbeat/README.md index 78014a62cb0..b2b0c14c8ac 100644 --- a/heartbeat/README.md +++ b/heartbeat/README.md @@ -35,8 +35,7 @@ To run Heartbeat with debugging output enabled, run: ### Update Each beat has a template for the mapping in elasticsearch and a documentation for the fields -which is automatically generated based on `etc/fields.yml`. -To generate etc/heartbeat.template.json and etc/heartbeat.asciidoc +which is automatically generated based on `fields.yml`. ``` make update diff --git a/heartbeat/heartbeat.full.yml b/heartbeat/heartbeat.full.yml index 7fe14781283..a30820786b9 100644 --- a/heartbeat/heartbeat.full.yml +++ b/heartbeat/heartbeat.full.yml @@ -346,20 +346,12 @@ output.elasticsearch: # so the final name is heartbeat-%{[beat.version]}. #template.name: "heartbeat" - # Path to template file - #template.path: "${path.config}/heartbeat.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, heartbeat checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/heartbeat.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/libbeat/.gitignore b/libbeat/.gitignore index 7f1d67065f6..e62f49faf22 100644 --- a/libbeat/.gitignore +++ b/libbeat/.gitignore @@ -25,8 +25,6 @@ _testmain.go .idea .jenkins -/libbeat.template-es2x.json /libbeat.yml /libbeat.full.yml -/libbeat.template.json /docs/fields.asciidoc diff --git a/libbeat/Makefile b/libbeat/Makefile index fdda89c9853..04ed8d9c489 100644 --- a/libbeat/Makefile +++ b/libbeat/Makefile @@ -14,3 +14,6 @@ fields: # Collects all dependencies and then calls update .PHONY: collect collect: fields + +clean:: + rm -rf libbeat.full.yml libbeat.yml diff --git a/libbeat/_meta/config.full.yml b/libbeat/_meta/config.full.yml index c58a78b3828..dfd7e493b44 100644 --- a/libbeat/_meta/config.full.yml +++ b/libbeat/_meta/config.full.yml @@ -148,20 +148,12 @@ output.elasticsearch: # so the final name is beatname-%{[beat.version]}. #template.name: "beatname" - # Path to template file - #template.path: "${path.config}/beatname.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, beatname checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/beatname.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index 5716483f4bb..7d9eef9f90e 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -23,7 +23,7 @@ Example configuration: output.elasticsearch: hosts: ["http://localhost:9200"] template.enabled: true - template.path: "{beatname_lc}.template.json" + template.fields: "fields.yml" template.overwrite: false index: "{beatname_lc}" ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] @@ -253,7 +253,7 @@ you must <>. *`name`*:: The name of the template. The default is +{beatname_lc}+. -*`path`*:: The path to the template file. The default is +{beatname_lc}.template.json+. If a relative +*`path`*:: The path to the template file. The default is +fields.yml+. If a relative path is set, it is considered relative to the config path. See the <> section for details. @@ -267,7 +267,7 @@ For example: output.elasticsearch: hosts: ["localhost:9200"] template.name: "{beatname_lc}" - template.path: "{beatname_lc}.template.json" + template.fields: "fields.yml" template.overwrite: false ---------------------------------------------------------------------- @@ -289,9 +289,8 @@ For example: ---------------------------------------------------------------------- output.elasticsearch: hosts: ["localhost:9200"] - template.path: "{beatname_lc}.template.json" + template.fields: "{beatname_lc}.template.json" template.overwrite: false - template.versions.2x.path: "{beatname_lc}.template-es2x.json ---------------------------------------------------------------------- ===== max_retries diff --git a/libbeat/docs/shared-template-load.asciidoc b/libbeat/docs/shared-template-load.asciidoc index 78462f860f9..7035026edb3 100644 --- a/libbeat/docs/shared-template-load.asciidoc +++ b/libbeat/docs/shared-template-load.asciidoc @@ -17,16 +17,16 @@ In Elasticsearch, {elasticsearch}/indices-templates.html[index -templates] are used to define settings and mappings that determine how fields should be analyzed. +templates] are used to define settings and mappings that determine how fields should be analyzed. The recommended index template file for {beatname_uc} is installed by the {beatname_uc} packages. If you accept -the default configuration for template loading in the +{beatname_lc}.yml+ config file, +the default configuration for template loading in the +{beatname_lc}.yml+ config file, {beatname_uc} loads the template automatically after successfully connecting to Elasticsearch. If the template already exists, it's not overwritten unless you configure {beatname_uc} to do so. -If you want to disable automatic template loading, or you want to load your own template, -you can change the settings for template loading in the {beatname_uc} configuration file. If you -choose to disable automatic template loading, you need to load the template manually. +If you want to disable automatic template loading, or you want to load your own template, +you can change the settings for template loading in the {beatname_uc} configuration file. If you +choose to disable automatic template loading, you need to load the template manually. For more information, see: * <> - supported for Elasticsearch output only @@ -35,16 +35,16 @@ For more information, see: [[load-template-auto]] ==== Configuring Template Loading -By default, {beatname_uc} automatically loads the recommended template file, +{beatname_lc}.template.json+, +By default, {beatname_uc} automatically loads the recommended template file, +fields.yml+, if Elasticsearch output is enabled. You can configure {beatname_lc} to load a different template -by adjusting the `template.name` and `template.path` options in +{beatname_lc}.yml+ file: +by adjusting the `template.name` and `template.fields` options in +{beatname_lc}.yml+ file: ["source","yaml",subs="attributes,callouts"] ---------------------------------------------------------------------- output.elasticsearch: hosts: ["localhost:9200"] template.name: "{beatname_lc}" - template.path: "{beatname_lc}.template.json" + template.fields: "fields.yml" template.overwrite: false ---------------------------------------------------------------------- diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 0a30c55c287..007c793677f 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -549,7 +549,6 @@ func itemStatusInner(reader *jsonReader) (int, []byte, error) { // PublishEvent publishes an event. func (client *Client) PublishEvent(data outputs.Data) error { // insert the events one by one - event := data.Event index := getIndex(event, client.index) @@ -596,6 +595,7 @@ func (client *Client) PublishEvent(data outputs.Data) error { // then use CheckTemplate prior to calling this method. func (client *Client) LoadTemplate(templateName string, template map[string]interface{}) error { + logp.Info("load template: %s", templateName) path := "/_template/" + templateName body, err := client.LoadJSON(path, template) if err != nil { diff --git a/libbeat/outputs/elasticsearch/client_integration_test.go b/libbeat/outputs/elasticsearch/client_integration_test.go index 426a8efb591..1b3cf27bcf2 100644 --- a/libbeat/outputs/elasticsearch/client_integration_test.go +++ b/libbeat/outputs/elasticsearch/client_integration_test.go @@ -4,16 +4,17 @@ package elasticsearch import ( "os" + "path/filepath" "strings" "testing" "time" - "path/filepath" - "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/libbeat/outputs" + "github.com/elastic/beats/libbeat/template" "github.com/elastic/beats/libbeat/version" + "github.com/stretchr/testify/assert" ) @@ -46,27 +47,26 @@ func TestLoadTemplate(t *testing.T) { assert.NotNil(t, absPath) assert.Nil(t, err) - templatePath := absPath + "/libbeat.template.json" - if strings.HasPrefix(client.Connection.version, "2.") { - templatePath = absPath + "/libbeat.template-es2x.json" - } - content, err := readTemplate(templatePath) - assert.Nil(t, err) + fieldsPath := absPath + "/fields.yml" + index := "testbeat" - templateName := "testbeat" + tmpl, err := template.New(version.GetDefaultVersion(), client.Connection.version, index) + assert.NoError(t, err) + content, err := tmpl.Load(fieldsPath) + assert.NoError(t, err) // Load template - err = client.LoadTemplate(templateName, content) + err = client.LoadTemplate(tmpl.GetName(), content) assert.Nil(t, err) // Make sure template was loaded - assert.True(t, client.CheckTemplate(templateName)) + assert.True(t, client.CheckTemplate(tmpl.GetName())) // Delete template again to clean up - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) // Make sure it was removed - assert.False(t, client.CheckTemplate(templateName)) + assert.False(t, client.CheckTemplate(tmpl.GetName())) } @@ -108,32 +108,29 @@ func TestLoadBeatsTemplate(t *testing.T) { // Setup ES client := GetTestingElasticsearch() - templatePath := absPath + "/" + beat + ".template.json" - - if strings.HasPrefix(client.Connection.version, "2.") { - templatePath = absPath + "/" + beat + ".template-es2x.json" - } - - content, err := readTemplate(templatePath) - assert.Nil(t, err) - err = client.Connect(5 * time.Second) assert.Nil(t, err) - templateName := beat + fieldsPath := absPath + "/fields.yml" + index := beat + + tmpl, err := template.New(version.GetDefaultVersion(), client.Connection.version, index) + assert.NoError(t, err) + content, err := tmpl.Load(fieldsPath) + assert.NoError(t, err) // Load template - err = client.LoadTemplate(templateName, content) + err = client.LoadTemplate(tmpl.GetName(), content) assert.Nil(t, err) // Make sure template was loaded - assert.True(t, client.CheckTemplate(templateName)) + assert.True(t, client.CheckTemplate(tmpl.GetName())) // Delete template again to clean up - client.Request("DELETE", "/_template/"+templateName, "", nil, nil) + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) // Make sure it was removed - assert.False(t, client.CheckTemplate(templateName)) + assert.False(t, client.CheckTemplate(tmpl.GetName())) } } @@ -147,28 +144,25 @@ func TestOutputLoadTemplate(t *testing.T) { t.Fatal(err) } + templateName := "libbeat-" + version.GetDefaultVersion() + // delete template if it exists - client.Request("DELETE", "/_template/libbeat", "", nil, nil) + client.Request("DELETE", "/_template/"+templateName, "", nil, nil) // Make sure template is not yet there - assert.False(t, client.CheckTemplate("libbeat")) - - templatePath := "../../libbeat.template.json" - - if strings.HasPrefix(client.Connection.version, "2.") { - templatePath = "../../libbeat.template-es2x.json" - } + assert.False(t, client.CheckTemplate(templateName)) + templatePath := "../../fields.yml" tPath, err := filepath.Abs(templatePath) if err != nil { t.Fatal(err) } + config := map[string]interface{}{ "hosts": GetEsHost(), "template": map[string]interface{}{ - "name": "libbeat", - "path": tPath, - "versions.2x.enabled": false, + "name": "libbeat", + "fields": tPath, }, } @@ -177,10 +171,11 @@ func TestOutputLoadTemplate(t *testing.T) { t.Fatal(err) } - output, err := New(common.BeatInfo{Beat: "libbeat"}, cfg) + output, err := New(common.BeatInfo{Beat: "libbeat", Version: version.GetDefaultVersion()}, cfg) if err != nil { t.Fatal(err) } + event := outputs.Data{Event: common.MapStr{ "@timestamp": common.Time(time.Now()), "host": "test-host", @@ -197,9 +192,7 @@ func TestOutputLoadTemplate(t *testing.T) { } // Guaranteed publish, so the template should be there - - assert.True(t, client.CheckTemplate("libbeat")) - + assert.True(t, client.CheckTemplate(templateName)) } func TestClientPublishEvent(t *testing.T) { diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go index 9d5a4281b2d..a3f12cb1dda 100644 --- a/libbeat/outputs/elasticsearch/config.go +++ b/libbeat/outputs/elasticsearch/config.go @@ -24,11 +24,10 @@ type elasticsearchConfig struct { // Template contains the elasticsearch template. type Template struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Path string `config:"path"` - Overwrite bool `config:"overwrite"` - Versions TemplateVersions `config:"versions"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` } // TemplateVersions contains the template versions. @@ -60,8 +59,8 @@ var ( TLS: nil, LoadBalance: true, Template: Template{ - Enabled: true, - Versions: TemplateVersions{Es2x: TemplateVersion{Enabled: true}}, + Enabled: true, + Fields: "fields.yml", }, } ) diff --git a/libbeat/outputs/elasticsearch/output.go b/libbeat/outputs/elasticsearch/output.go index 16503bc2d19..38a25d60aef 100644 --- a/libbeat/outputs/elasticsearch/output.go +++ b/libbeat/outputs/elasticsearch/output.go @@ -1,12 +1,9 @@ package elasticsearch import ( - "encoding/json" "errors" "fmt" "net/url" - "os" - "strings" "sync" "time" @@ -19,6 +16,7 @@ import ( "github.com/elastic/beats/libbeat/outputs/outil" "github.com/elastic/beats/libbeat/outputs/transport" "github.com/elastic/beats/libbeat/paths" + "github.com/elastic/beats/libbeat/template" ) type elasticsearchOutput struct { @@ -29,8 +27,6 @@ type elasticsearchOutput struct { mode mode.ConnectionMode - template map[string]interface{} - template2x map[string]interface{} templateMutex sync.Mutex } @@ -180,11 +176,6 @@ func (out *elasticsearchOutput) init( return err } - err = out.readTemplate(&config.Template) - if err != nil { - return err - } - out.index = index pipeline, err := outil.BuildSelectorFromConfig(cfg, outil.Settings{ Key: "pipeline", @@ -232,62 +223,6 @@ func (out *elasticsearchOutput) init( return nil } -// readTemplates reads the ES mapping template from the disk, if configured. -func (out *elasticsearchOutput) readTemplate(config *Template) error { - if config.Enabled { - // Set the defaults that depend on the beat name - if config.Name == "" { - config.Name = out.beat.Beat + "-" + out.beat.Version - } - if config.Path == "" { - config.Path = fmt.Sprintf("%s.template.json", out.beat.Beat) - } - if config.Versions.Es2x.Path == "" { - config.Versions.Es2x.Path = fmt.Sprintf("%s.template-es2x.json", out.beat.Beat) - } - - // Look for the template in the configuration path, if it's not absolute - templatePath := paths.Resolve(paths.Config, config.Path) - logp.Info("Loading template enabled. Reading template file: %v", templatePath) - - template, err := readTemplate(templatePath) - if err != nil { - return fmt.Errorf("Error loading template %s: %v", templatePath, err) - } - out.template = template - - if config.Versions.Es2x.Enabled { - // Read the version of the template compatible with ES 2.x - templatePath := paths.Resolve(paths.Config, config.Versions.Es2x.Path) - logp.Info("Loading template enabled for Elasticsearch 2.x. Reading template file: %v", templatePath) - - template, err := readTemplate(templatePath) - if err != nil { - return fmt.Errorf("Error loading template %s: %v", templatePath, err) - } - out.template2x = template - } - } - return nil -} - -func readTemplate(filename string) (map[string]interface{}, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - defer f.Close() - - var template map[string]interface{} - dec := json.NewDecoder(f) - err = dec.Decode(&template) - if err != nil { - return nil, err - } - - return template, nil -} - // loadTemplate checks if the index mapping template should be loaded // In case the template is not already loaded or overwriting is enabled, the // template is written to index @@ -301,19 +236,27 @@ func (out *elasticsearchOutput) loadTemplate(config Template, client *Client) er exists := client.CheckTemplate(config.Name) if !exists || config.Overwrite { + logp.Info("Loading template for elasticsearch version: %s", client.Connection.version) + if config.Overwrite { logp.Info("Existing template will be overwritten, as overwrite is enabled.") } - template := out.template - if config.Versions.Es2x.Enabled && strings.HasPrefix(client.Connection.version, "2.") { - logp.Info("Detected Elasticsearch 2.x. Automatically selecting the 2.x version of the template") - template = out.template2x + tmpl, err := template.New(out.beat.Version, client.Connection.version, config.Name) + if err != nil { + return fmt.Errorf("error creating template instance: %v", err) } - err := client.LoadTemplate(config.Name, template) + fieldsPath := paths.Resolve(paths.Config, config.Fields) + + output, err := tmpl.Load(fieldsPath) if err != nil { - return fmt.Errorf("Could not load template: %v", err) + return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) + } + + err = client.LoadTemplate(tmpl.GetName(), output) + if err != nil { + return fmt.Errorf("could not load template: %v", err) } } else { logp.Info("Template already exists and will not be overwritten.") @@ -349,9 +292,14 @@ func makeClientFactory( params = nil } - // define a callback to be called on connection var onConnected connectCallback - if out.template != nil { + // TODO: should we check if fields.yml exists? + if config.Template.Enabled { + // Set beat name as default if name not set + if config.Template.Name == "" { + config.Template.Name = out.beat.Beat + } + // define a callback to be called on connection onConnected = func(client *Client) error { return out.loadTemplate(config.Template, client) } diff --git a/libbeat/paths/paths.go b/libbeat/paths/paths.go index 638f9afc093..971aad8549e 100644 --- a/libbeat/paths/paths.go +++ b/libbeat/paths/paths.go @@ -127,6 +127,7 @@ func (paths *Path) Resolve(fileType FileType, path string) string { // Resolve resolves a path to a location in one of the default // folders. For example, Resolve(Home, "test") returns an absolute // path for "test" in the home path. +// In case path is already an absolute path, the path itself is returned. func Resolve(fileType FileType, path string) string { return Paths.Resolve(fileType, path) } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 14334feec19..2e451dd1fc1 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -107,6 +107,8 @@ clean:: ## @build Cleans up all files generated by the build steps rm -f ${BEAT_NAME}.yml {BEAT_NAME}.full.yml # Clean index pattern rm -f $(PWD)/_meta/kibana/index-pattern/${BEAT_NAME}.json + # Remove temp templates + -rm ${BEAT_NAME}.template*.json .PHONY: ci ci: ## @build Shortcut for continuous integration. This should always run before merging. @@ -261,10 +263,6 @@ update: python-env collect mkdir -p docs . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS} - # Generate index templates - go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml - go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml - # Generate index-pattern echo "Generate index pattern" mkdir -p $(PWD)/_meta/kibana/index-pattern @@ -280,6 +278,9 @@ docs-preview: ## @build Preview the documents for the beat in the browser if [ ! -d "build/docs" ]; then $(MAKE) docs; fi; ${BUILD_DIR}/docs/build_docs.pl --chunk=1 -open chunk=1 -open --doc ${BEAT_GOPATH}/src/github.com/elastic/beats/${BEAT_NAME}/docs/index.asciidoc -out ${BUILD_DIR}/html_docs +.PHONY: index-template +index-template: ## @build Generate index templates for the given $VERSION. This is for manual testing. + go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version ${VERSION} -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es${VERSION}.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml ### KIBANA FILES HANDLING ### ES_URL?=http://localhost:9200 diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 8fb15639841..e2811154ac1 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -480,20 +480,12 @@ output.elasticsearch: # so the final name is metricbeat-%{[beat.version]}. #template.name: "metricbeat" - # Path to template file - #template.path: "${path.config}/metricbeat.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, metricbeat checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/metricbeat.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/packetbeat/packetbeat.full.yml b/packetbeat/packetbeat.full.yml index e4077259d47..24b5b64a1a6 100644 --- a/packetbeat/packetbeat.full.yml +++ b/packetbeat/packetbeat.full.yml @@ -603,20 +603,12 @@ output.elasticsearch: # so the final name is packetbeat-%{[beat.version]}. #template.name: "packetbeat" - # Path to template file - #template.path: "${path.config}/packetbeat.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, packetbeat checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/packetbeat.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true diff --git a/winlogbeat/winlogbeat.full.yml b/winlogbeat/winlogbeat.full.yml index 171a178bfac..5851054c52d 100644 --- a/winlogbeat/winlogbeat.full.yml +++ b/winlogbeat/winlogbeat.full.yml @@ -183,20 +183,12 @@ output.elasticsearch: # so the final name is winlogbeat-%{[beat.version]}. #template.name: "winlogbeat" - # Path to template file - #template.path: "${path.config}/winlogbeat.template.json" + # Path to fields.yml file to generate the template + #template.fields: "${path.config}/fields.yml" # Overwrite existing template #template.overwrite: false - # If set to true, winlogbeat checks the Elasticsearch version at connect time, and if it - # is 2.x, it loads the file specified by the template.versions.2x.path setting. The - # default is true. - #template.versions.2x.enabled: true - - # Path to the Elasticsearch 2.x version of the template file. - #template.versions.2x.path: "${path.config}/winlogbeat.template-es2x.json" - # Use SSL settings for HTTPS. Default is true. #ssl.enabled: true