From 5bcd42e6c6b88dfcdfcab78d8f79937858770f40 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Wed, 10 May 2017 23:42:17 +0200 Subject: [PATCH] Template setting in the configuration file This allows adding and overwriting index/template settings via the configuration file. Example config: setup.template.settings: index.number_of_shards: 1 index.number_of_replicas: 1 Needed for #4112, part of #3654. --- CHANGELOG.asciidoc | 2 + libbeat/docs/index.asciidoc | 1 + libbeat/docs/template-config.asciidoc | 13 +++-- libbeat/template/config.go | 12 ++--- libbeat/template/load.go | 2 +- libbeat/template/load_integration_test.go | 60 ++++++++++++++++++++++- libbeat/template/template.go | 27 ++++++---- 7 files changed, 95 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 952e6aa8d76..d6bc616a38e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* +- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] + *Filebeat* *Heartbeat* diff --git a/libbeat/docs/index.asciidoc b/libbeat/docs/index.asciidoc index fda42672f70..690b0fd2c7f 100644 --- a/libbeat/docs/index.asciidoc +++ b/libbeat/docs/index.asciidoc @@ -11,6 +11,7 @@ include::./version.asciidoc[] :beatsdevguide: http://www.elastic.co/guide/en/beats/devguide/{doc-branch} :securitydoc: https://www.elastic.co/guide/en/x-pack/5.2 :logstashdoc: https://www.elastic.co/guide/en/logstash/{doc-branch} +:elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/{doc-branch} :beatname_lc: beatname :beatname_uc: a Beat :security: X-Pack Security diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index c046922b9e4..3c758c56e15 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -13,13 +13,17 @@ you must <>. *`name`*:: The name of the template. The default is +{beatname_lc}+. -*`path`*:: The path to the template file. The default is +fields.yml+. If a relative -path is set, it is considered relative to the config path. See the <> section for -details. +*`fields`*:: The path to the YAML file describing the fields. The default is +fields.yml+. If a +relative path is set, it is considered relative to the config path. See the <> +section for details. *`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default is false. +*`settings`*:: A dictionary of settings to place into the `settings` dictionary of the Elasticsearch +template. For more details about the available Elasticsearch mapping options, please see the +Elasticsearch {elasticsearch}/mapping.html[mapping reference]. + For example: ["source","yaml",subs="attributes,callouts"] @@ -27,4 +31,7 @@ For example: setup.template.name: "{beatname_lc}" setup.template.fields: "fields.yml" setup.template.overwrite: false +setup.template.settings: + index.number_of_shards: 1 + index.number_of_replicas: 1 ---------------------------------------------------------------------- diff --git a/libbeat/template/config.go b/libbeat/template/config.go index 7be6f7702ab..fe6185972e2 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,12 +1,12 @@ package template type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - OutputToFile string `config:"output_to_file"` - Settings map[string]string `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + OutputToFile string `config:"output_to_file"` + Settings map[string]interface{} `config:"settings"` } var ( diff --git a/libbeat/template/load.go b/libbeat/template/load.go index d6ac02c1dea..b1b49b9c0b5 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -57,7 +57,7 @@ func (l *Loader) Load() error { l.config.Name = l.beatInfo.Beat } - tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name) + tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings) if err != nil { return fmt.Errorf("error creating template instance: %v", err) } diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 75e89464e7a..579f1d912a8 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -3,10 +3,13 @@ package template import ( + "encoding/json" + "fmt" "path/filepath" "testing" "time" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/outputs/elasticsearch" "github.com/elastic/beats/libbeat/version" @@ -43,7 +46,7 @@ func TestLoadTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := "testbeat" - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index) + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -114,7 +117,7 @@ func TestLoadBeatsTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := beat - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index) + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -137,3 +140,56 @@ func TestLoadBeatsTemplate(t *testing.T) { assert.False(t, loader.CheckTemplate(tmpl.GetName())) } } + +func TestTemplateSettings(t *testing.T) { + + // Setup ES + client := elasticsearch.GetTestingElasticsearch() + err := client.Connect(5 * time.Second) + assert.Nil(t, err) + + // Load template + absPath, err := filepath.Abs("../") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + fieldsPath := absPath + "/fields.yml" + + settings := common.MapStr{ + "index": common.MapStr{ + "number_of_shards": 1, + }, + } + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings) + assert.NoError(t, err) + content, err := tmpl.Load(fieldsPath) + assert.NoError(t, err) + + loader := &Loader{ + client: client, + } + + // Load template + err = loader.LoadTemplate(tmpl.GetName(), content) + assert.Nil(t, err) + + // Check that it contains the mapping + status, body, err := loader.client.Request("GET", "/_template/"+tmpl.GetName(), "", nil, nil) + assert.NoError(t, err) + assert.Equal(t, status, 200) + + var response common.MapStr + err = json.Unmarshal(body, &response) + assert.NoError(t, err) + + templateJSON := common.MapStr(response[tmpl.GetName()].(map[string]interface{})) + val, err := templateJSON.GetValue(fmt.Sprintf("settings.index.number_of_shards")) + assert.NoError(t, err) + assert.Equal(t, val.(string), "1") + + // Delete template again to clean up + client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) + + // Make sure it was removed + assert.False(t, loader.CheckTemplate(tmpl.GetName())) +} diff --git a/libbeat/template/template.go b/libbeat/template/template.go index f2ad6b7f04a..602baf40f8f 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -20,10 +20,11 @@ type Template struct { index string beatVersion Version esVersion Version + settings common.MapStr } // New creates a new template instance -func New(beatVersion string, esVersion string, index string) (*Template, error) { +func New(beatVersion string, esVersion string, index string, settings common.MapStr) (*Template, error) { bV, err := NewVersion(beatVersion) if err != nil { @@ -44,6 +45,7 @@ func New(beatVersion string, esVersion string, index string) (*Template, error) index: index, beatVersion: *bV, esVersion: *esV, + settings: settings, }, nil } @@ -90,6 +92,18 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase) + settings := common.MapStr{ + "index": common.MapStr{ + "refresh_interval": "5s", + "mapping": common.MapStr{ + "total_fields": common.MapStr{ + "limit": defaultTotalFieldsLimit, + }, + }, + }, + } + settings.DeepUpdate(t.settings) + // Load basic structure basicStructure := common.MapStr{ "mappings": common.MapStr{ @@ -102,10 +116,8 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. "properties": properties, }, }, - "order": 1, - "settings": common.MapStr{ - "index.refresh_interval": "5s", - }, + "order": 1, + "settings": settings, } // ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009 @@ -119,11 +131,6 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. basicStructure.Put("mappings._default_._all.norms.enabled", false) } - if t.esVersion.major >= 5 { - // Metricbeat exceeds the default of 1000 fields - basicStructure.Put("settings.index.mapping.total_fields.limit", defaultTotalFieldsLimit) - } - return basicStructure }