From d7d5e0af9d559790068ec5cdef1e01abd574501d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Mon, 26 Jun 2017 12:21:28 +0200 Subject: [PATCH] Remove `output_to_file` template settings Deprecated in favor of `beat export template` --- libbeat/beat/beat.go | 14 --- libbeat/beat/setup.go | 11 +-- libbeat/docs/template-config.asciidoc | 22 ----- libbeat/template/config.go | 18 +--- libbeat/template/load.go | 39 -------- libbeat/template/load_test.go | 135 -------------------------- libbeat/tests/system/test_template.py | 47 --------- 7 files changed, 10 insertions(+), 276 deletions(-) delete mode 100644 libbeat/template/load_test.go delete mode 100644 libbeat/tests/system/test_template.py diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index fba1b94426a..56802e8a2b5 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -600,20 +600,6 @@ func (b *Beat) registerTemplateLoading() error { if err != nil { return fmt.Errorf("unpacking template config fails: %v", err) } - if len(cfg.OutputToFile.Path) > 0 { - // output to file is enabled - loader, err := template.NewLoader(b.Config.Template, nil, b.Info) - if err != nil { - return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) - } - err = loader.Generate() - if err != nil { - return fmt.Errorf("Error generating template: %v", err) - } - - // XXX: Should we kill the Beat here or just continue? - return fmt.Errorf("Stopping after successfully writing the template to the file.") - } } // Loads template by default if esOutput is enabled diff --git a/libbeat/beat/setup.go b/libbeat/beat/setup.go index 061e9091270..1cb5c2d4a41 100644 --- a/libbeat/beat/setup.go +++ b/libbeat/beat/setup.go @@ -1,10 +1,9 @@ package beat type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - OutputToFile string `config:"output_to_file"` - Settings map[string]string `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + Settings map[string]string `config:"settings"` } diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 3a554590dd1..a27b2fd9ebb 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -49,25 +49,3 @@ setup.template.overwrite: false setup.template.settings: _source.enabled: false ---------------------------------------------------------------------- - -*`output_to_file.path`*:: If this option is set, {beatname_uc} generates the Elasticsearch template - JSON object and writes into a file at the specified path. Immediately after writing the file, - {beatname_uc} exists with the exit code 1. - -For example, you can generate a template file ready to be uploaded to Elasticsearch like this: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json" ----------------------------------------------------------------------- - -*`output_to_file.version`*:: The Elasticsearch version for which to generate the template file. By -default, the {beatname_uc} version is used. This setting is only used if `output_to_file.path` is -also set. - -For example, the following generates a template file for Elasticsearch 5.4: - -["source","yaml",subs="attributes,callouts"] ----------------------------------------------------------------------- -./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json" -E "setup.template.output_to_file.version=5.4.0" ----------------------------------------------------------------------- diff --git a/libbeat/template/config.go b/libbeat/template/config.go index bb0f401ffd4..9aacc7e997a 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,19 +1,11 @@ package template type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - Settings TemplateSettings `config:"settings"` - OutputToFile OutputToFile `config:"output_to_file"` -} - -// OutputToFile contains the configuration options for generating -// and writing the template into a file. -type OutputToFile struct { - Path string `config:"path"` - Version string `config:"version"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + Settings TemplateSettings `config:"settings"` } type TemplateSettings struct { diff --git a/libbeat/template/load.go b/libbeat/template/load.go index e1bf6f86e49..d362c7dfd84 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -1,9 +1,7 @@ package template import ( - "encoding/json" "fmt" - "io/ioutil" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -82,43 +80,6 @@ func (l *Loader) Load() error { return nil } -// Generate generates the template and writes it to a file based on the configuration -// from `output_to_file`. -func (l *Loader) Generate() error { - if l.config.OutputToFile.Version == "" { - l.config.OutputToFile.Version = l.beatInfo.Version - } - - if l.config.Name == "" { - l.config.Name = l.beatInfo.Beat - } - - tmpl, err := New(l.beatInfo.Version, l.config.OutputToFile.Version, l.config.Name, l.config.Settings) - if err != nil { - return fmt.Errorf("error creating template instance: %v", err) - } - - fieldsPath := paths.Resolve(paths.Config, l.config.Fields) - - output, err := tmpl.Load(fieldsPath) - if err != nil { - return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) - } - - jsonBytes, err := json.MarshalIndent(output, "", " ") - if err != nil { - return fmt.Errorf("error marshaling template: %v", err) - } - - err = ioutil.WriteFile(l.config.OutputToFile.Path, jsonBytes, 0644) - if err != nil { - return fmt.Errorf("error writing to file %s: %v", l.config.OutputToFile.Path, err) - } - - logp.Info("Template for Elasticsearch %s written to: %s", l.config.OutputToFile.Version, l.config.OutputToFile.Path) - return nil -} - // LoadTemplate loads a template into Elasticsearch overwriting the existing // template if it exists. If you wish to not overwrite an existing template // then use CheckTemplate prior to calling this method. diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go deleted file mode 100644 index d6347d66387..00000000000 --- a/libbeat/template/load_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// +build !integration - -package template - -import ( - "encoding/json" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/version" - "github.com/stretchr/testify/assert" -) - -func TestGenerateTemplate(t *testing.T) { - // Load template - absPath, err := filepath.Abs("../") - if err != nil { - t.Fatal(err) - } - - beatInfo := common.BeatInfo{ - Beat: "testbeat", - Version: version.GetDefaultVersion(), - } - - dir, err := ioutil.TempDir("", "test-template") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - outputFile := filepath.Join(dir, "template.json") - - config := newConfigFrom(t, TemplateConfig{ - Enabled: true, - Fields: filepath.Join(absPath, "fields.yml"), - OutputToFile: OutputToFile{ - Path: outputFile, - }, - }) - - loader, err := NewLoader(config, nil, beatInfo) - if err != nil { - t.Fatal(err) - } - - if err = loader.Generate(); err != nil { - t.Fatal("generate failed", err) - } - - // Read it back to check it - fp, err := os.Open(outputFile) - if err != nil { - t.Fatal(err) - } - - jsonParser := json.NewDecoder(fp) - var parsed common.MapStr - if err = jsonParser.Decode(&parsed); err != nil { - t.Fatal("decoding failed", err) - } - - val, err := parsed.GetValue("mappings._default_._meta.version") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, val.(string), version.GetDefaultVersion()) -} - -func TestGenerateTemplateWithVersion(t *testing.T) { - // Load template - absPath, err := filepath.Abs("../") - if err != nil { - t.Fatal(err) - } - - beatInfo := common.BeatInfo{ - Beat: "testbeat", - Version: version.GetDefaultVersion(), - } - - dir, err := ioutil.TempDir("", "test-template") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - outputFile := filepath.Join(dir, "template.json") - - config := newConfigFrom(t, TemplateConfig{ - Enabled: true, - Fields: filepath.Join(absPath, "fields.yml"), - OutputToFile: OutputToFile{ - Path: outputFile, - Version: "2.4.0", - }, - }) - - loader, err := NewLoader(config, nil, beatInfo) - if err != nil { - t.Fatal(err) - } - - if err = loader.Generate(); err != nil { - t.Fatal("generate failed", err) - } - - // Read it back to check it - fp, err := os.Open(outputFile) - if err != nil { - t.Fatal(err) - } - - jsonParser := json.NewDecoder(fp) - var parsed common.MapStr - if err = jsonParser.Decode(&parsed); err != nil { - t.Fatal("decoding failed", err) - } - - // Check a setting specific to that version. - val, err := parsed.GetValue("mappings._default_._all.norms.enabled") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, val.(bool), false) -} - -func newConfigFrom(t *testing.T, from interface{}) *common.Config { - cfg, err := common.NewConfigFrom(from) - assert.NoError(t, err) - return cfg -} diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py deleted file mode 100644 index 41ea5359124..00000000000 --- a/libbeat/tests/system/test_template.py +++ /dev/null @@ -1,47 +0,0 @@ -from base import BaseTest - -import os -import json - - -class Test(BaseTest): - - def test_generate_templates(self): - """ - Generates templates from other Beats. - """ - self.render_config_template() - - output_json = os.path.join(self.working_dir, "template.json") - fields_yml = "../../../../fields.yml" - - exit_code = self.run_beat(extra_args=[ - "-E", "setup.template.output_to_file.path={}".format(output_json), - "-E", "setup.template.fields={}".format(fields_yml)]) - assert exit_code == 1 - - # check json file - with open(output_json) as f: - tmpl = json.load(f) - assert "mappings" in tmpl - - def test_generate_templates_v5(self): - """ - Generates templates from other Beats. - """ - self.render_config_template() - - output_json = os.path.join(self.working_dir, "template-5x.json") - fields_yml = "../../../../fields.yml" - - exit_code = self.run_beat(extra_args=[ - "-E", "setup.template.output_to_file.path={}".format(output_json), - "-E", "setup.template.output_to_file.version=5.0.0".format(output_json), - "-E", "setup.template.fields={}".format(fields_yml)]) - assert exit_code == 1 - - # check json file - with open(output_json) as f: - tmpl = json.load(f) - assert "mappings" in tmpl - assert tmpl["mappings"]["_default_"]["_all"]["norms"]["enabled"] is False