From 80c8cd0dab3bc3446085183599819a09aef8f4e2 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 28 Feb 2017 01:37:53 +0100 Subject: [PATCH] Simplify template generation and handling (#3683) * Only one file is passed instead of an array as now only file is needed. Introduce `-file` param. * Local loading of beats version was removed as it can lean to cyclic imports * Move `_meta/fields.full.generated.yml` to `fields.yml` in preparation for packaging `fields.yml` and ignore it in git * Cleanup Template generation package * Rename -beat.name param to -index as this describes it better Part of https://github.com/elastic/beats/issues/3654 --- .gitignore | 1 + .../cmd/index_template/index_template.go | 35 +++++----- libbeat/scripts/Makefile | 8 +-- libbeat/scripts/generate_fields_docs.py | 2 +- libbeat/scripts/generate_index_pattern.py | 2 +- libbeat/template/template.go | 64 ++++++++++++------- 6 files changed, 65 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index a09ac62818d..3737bf062a2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /*/data /*/logs /*/_meta/kibana/index-pattern +/*/fields.yml # Files .DS_Store diff --git a/dev-tools/cmd/index_template/index_template.go b/dev-tools/cmd/index_template/index_template.go index a9a086ecec8..8308c93f760 100644 --- a/dev-tools/cmd/index_template/index_template.go +++ b/dev-tools/cmd/index_template/index_template.go @@ -13,32 +13,27 @@ import ( // main generates index templates for the beats func main() { - beatName := flag.String("beat.name", "", ": Base index name. Normally {beat_name} (required)") + beatVersion := beat.GetDefaultVersion() + index := flag.String("index", "", "Base index name. Normally {beat_name} (required)") output := flag.String("output", "", "Required: Full path to the output file (required)") - version := flag.String("es.version", beat.GetDefaultVersion(), "Elasticsearch version") + version := flag.String("es.version", beatVersion, "Elasticsearch version") + file := flag.String("file", "", "Path to fields.yml file") flag.Parse() - var existingFiles []string - files := flag.Args() - - if len(files) == 0 { - fmt.Fprintf(os.Stderr, "No fields.yml files provided. At least one file must be added.") + if len(*file) == 0 { + fmt.Fprintf(os.Stderr, "File path cannot be empty") os.Exit(1) } - if *beatName == "" { - fmt.Fprintf(os.Stderr, "beat.name is empty. It must be set.") + if *index == "" { + fmt.Fprintf(os.Stderr, "index is empty. It must be set.") os.Exit(1) } - // Skip some of the passed files, as not all beats have the same files - for _, f := range files { - if _, err := os.Stat(f); err != nil { - fmt.Printf("Skipping file because it does not exist: %s", f) - continue - } - existingFiles = append(existingFiles, f) + if _, err := os.Stat(*file); err != nil { + fmt.Fprintf(os.Stderr, "Error during loading -file %s with error: %s", *file, err) + os.Exit(1) } // Make it compatible with the sem versioning @@ -46,7 +41,13 @@ func main() { *version = "2.0.0" } - templateString, err := template.GetTemplate(*version, *beatName, existingFiles) + tmpl, err := template.New(beatVersion, *version, *index) + if err != nil { + fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) + os.Exit(1) + } + + templateString, err := tmpl.Load(*file) if err != nil { fmt.Fprintf(os.Stderr, "Error generating template: %+v", err) os.Exit(1) diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index a9428694ad4..d85078a5263 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -247,17 +247,17 @@ update: python-env collect # Generate full fields for each beat. Create fields.generated.yml in case it does not exist yet. # Most beats have their own way of generating it. test -s _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml - cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > _meta/fields.full.generated.yml + cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml # Revert changes to libbeat fields as it otherwise contains duplicates in full - cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/_meta/fields.full.generated.yml + cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/fields.yml # Update docs mkdir -p docs . ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS} # Generate index templates - go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml - go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml + go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml + go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml # Generate index-pattern echo "Generate index pattern" diff --git a/libbeat/scripts/generate_fields_docs.py b/libbeat/scripts/generate_fields_docs.py index 7b9d82b5a15..6bae788bcc2 100644 --- a/libbeat/scripts/generate_fields_docs.py +++ b/libbeat/scripts/generate_fields_docs.py @@ -111,7 +111,7 @@ def fields_to_asciidoc(input, output, beat): beat_name = args.beatname es_beats = args.es_beats - fields_yml = beat_path + "/_meta/fields.full.generated.yml" + fields_yml = beat_path + "/fields.yml" # Read fields.yml with open(fields_yml) as f: diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index c7c2735de5a..e31c8f9cb76 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -139,7 +139,7 @@ def get_index_pattern_name(index): args = parser.parse_args() - fields_yml = args.beat + "/_meta/fields.full.generated.yml" + fields_yml = args.beat + "/fields.yml" # generate the index-pattern content with open(fields_yml, 'r') as f: diff --git a/libbeat/template/template.go b/libbeat/template/template.go index a51859a7d53..ec63fe03eab 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -3,7 +3,6 @@ package template import ( "fmt" - "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/go-ucfg/yaml" ) @@ -17,44 +16,61 @@ var ( dynamicTemplates []common.MapStr ) -// GetTemplate creates a template based on the given inputs -func GetTemplate(version string, beatName string, files []string) (common.MapStr, error) { +type Template struct { + index string + beatVersion Version + esVersion Version +} - beatVersion := beat.GetDefaultVersion() +// New creates a new template instance +func New(beatVersion string, esVersion string, index string) (*Template, error) { + + bV, err := NewVersion(beatVersion) + if err != nil { + return nil, err + } // In case no esVersion is set, it is assumed the same as beat version - if version == "" { - version = beatVersion + if esVersion == "" { + esVersion = beatVersion } - esVersion, err := NewVersion(version) + esV, err := NewVersion(esVersion) if err != nil { return nil, err } - fields := Fields{} + return &Template{ + index: index, + beatVersion: *bV, + esVersion: *esV, + }, nil - for _, file := range files { - f, err := loadYaml(file) - if err != nil { - return nil, err - } - fields = append(fields, f...) +} + +// Load the given input and generates the input based on it +func (t *Template) Load(file string) (common.MapStr, error) { + + fields, err := loadYaml(file) + if err != nil { + return nil, err } // Start processing at the root - properties := fields.process("", *esVersion) - - indexPattern := fmt.Sprintf("%s-%s-*", beatName, beatVersion) - output := createTemplate(properties, beatVersion, *esVersion, indexPattern, dynamicTemplates) + properties := fields.process("", t.esVersion) + output := t.generate(properties, dynamicTemplates) return output, nil +} +// GetName returns the name of the template which is {index}-{version} +func (t *Template) GetName() string { + return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String()) } -// createTemplate creates the full template +// generate generates the full template // The default values are taken from the default variable. -func createTemplate(properties common.MapStr, version string, esVersion Version, indexPattern string, dynamicTemplates []common.MapStr) common.MapStr { +func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { // Add base dynamic template var dynamicTemplateBase = common.MapStr{ @@ -67,7 +83,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version, }, } - if esVersion.IsMajor(2) { + if t.esVersion.IsMajor(2) { dynamicTemplateBase.Put("strings_as_keyword.mapping.type", "string") dynamicTemplateBase.Put("strings_as_keyword.mapping.index", "not_analyzed") } @@ -79,7 +95,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version, "mappings": common.MapStr{ "_default_": common.MapStr{ "_meta": common.MapStr{ - "version": version, + "version": t.beatVersion.String(), }, "date_detection": defaultDateDetection, "dynamic_templates": dynamicTemplates, @@ -90,10 +106,10 @@ func createTemplate(properties common.MapStr, version string, esVersion Version, "settings": common.MapStr{ "index.refresh_interval": "5s", }, - "template": indexPattern, + "template": t.GetName() + "-*", } - if esVersion.IsMajor(2) { + if t.esVersion.IsMajor(2) { basicStructure.Put("mappings._default_._all.norms.enabled", false) } else { // Metricbeat exceeds the default of 1000 fields