Simplify template generation and handling (#3683)

* Only one file is passed instead of an array as now only file is needed. Introduce `-file` param.
* Local loading of beats version was removed as it can lean to cyclic imports
* Move `_meta/fields.full.generated.yml` to `fields.yml` in preparation for packaging `fields.yml` and ignore it in git
* Cleanup Template generation package
* Rename -beat.name param to -index as this describes it better

Part of #3654

.gitignore

@@ -5,6 +5,7 @@
 /*/data
 /*/logs
 /*/_meta/kibana/index-pattern
+/*/fields.yml
 
 # Files
 .DS_Store

dev-tools/cmd/index_template/index_template.go

@@ -13,40 +13,41 @@ import (
 // main generates index templates for the beats
 func main() {
 
-	beatName := flag.String("beat.name", "", ": Base index name. Normally {beat_name} (required)")
+	beatVersion := beat.GetDefaultVersion()
+	index := flag.String("index", "", "Base index name. Normally {beat_name} (required)")
 	output := flag.String("output", "", "Required: Full path to the output file (required)")
-	version := flag.String("es.version", beat.GetDefaultVersion(), "Elasticsearch version")
+	version := flag.String("es.version", beatVersion, "Elasticsearch version")
+	file := flag.String("file", "", "Path to fields.yml file")
 
 	flag.Parse()
 
-	var existingFiles []string
-	files := flag.Args()
-
-	if len(files) == 0 {
-		fmt.Fprintf(os.Stderr, "No fields.yml files provided. At least one file must be added.")
+	if len(*file) == 0 {
+		fmt.Fprintf(os.Stderr, "File path cannot be empty")
 		os.Exit(1)
 	}
 
-	if *beatName == "" {
-		fmt.Fprintf(os.Stderr, "beat.name is empty. It must be set.")
+	if *index == "" {
+		fmt.Fprintf(os.Stderr, "index is empty. It must be set.")
 		os.Exit(1)
 	}
 
-	// Skip some of the passed files, as not all beats have the same files
-	for _, f := range files {
-		if _, err := os.Stat(f); err != nil {
-			fmt.Printf("Skipping file because it does not exist: %s", f)
-			continue
-		}
-		existingFiles = append(existingFiles, f)
+	if _, err := os.Stat(*file); err != nil {
+		fmt.Fprintf(os.Stderr, "Error during loading -file %s with error: %s", *file, err)
+		os.Exit(1)
 	}
 
 	// Make it compatible with the sem versioning
 	if *version == "2x" {
 		*version = "2.0.0"
 	}
 
-	templateString, err := template.GetTemplate(*version, *beatName, existingFiles)
+	tmpl, err := template.New(beatVersion, *version, *index)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
+		os.Exit(1)
+	}
+
+	templateString, err := tmpl.Load(*file)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
 		os.Exit(1)

libbeat/scripts/Makefile

@@ -247,17 +247,17 @@ update: python-env collect
 	# Generate full fields for each beat. Create fields.generated.yml in case it does not exist yet.
 	# Most beats have their own way of generating it.
 	test -s  _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
-	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > _meta/fields.full.generated.yml
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > fields.yml
 	# Revert changes to libbeat fields as it otherwise contains duplicates in full
-	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/_meta/fields.full.generated.yml
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/fields.yml
 
 	# Update docs
 	mkdir -p docs
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS}
 
 	# Generate index templates
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -index ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json -file ${BEAT_GOPATH}/src/${BEAT_PATH}/fields.yml
 
 	# Generate index-pattern
 	echo "Generate index pattern"

libbeat/scripts/generate_fields_docs.py

@@ -111,7 +111,7 @@ def fields_to_asciidoc(input, output, beat):
     beat_name = args.beatname
     es_beats = args.es_beats
 
-    fields_yml = beat_path + "/_meta/fields.full.generated.yml"
+    fields_yml = beat_path + "/fields.yml"
 
     # Read fields.yml
     with open(fields_yml) as f:

libbeat/scripts/generate_index_pattern.py

@@ -139,7 +139,7 @@ def get_index_pattern_name(index):
 
     args = parser.parse_args()
 
-    fields_yml = args.beat + "/_meta/fields.full.generated.yml"
+    fields_yml = args.beat + "/fields.yml"
 
     # generate the index-pattern content
     with open(fields_yml, 'r') as f:

libbeat/template/template.go

@@ -3,7 +3,6 @@ package template
 import (
 	"fmt"
 
-	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/go-ucfg/yaml"
 )
@@ -17,44 +16,61 @@ var (
 	dynamicTemplates []common.MapStr
 )
 
-// GetTemplate creates a template based on the given inputs
-func GetTemplate(version string, beatName string, files []string) (common.MapStr, error) {
+type Template struct {
+	index       string
+	beatVersion Version
+	esVersion   Version
+}
 
-	beatVersion := beat.GetDefaultVersion()
+// New creates a new template instance
+func New(beatVersion string, esVersion string, index string) (*Template, error) {
+
+	bV, err := NewVersion(beatVersion)
+	if err != nil {
+		return nil, err
+	}
 
 	// In case no esVersion is set, it is assumed the same as beat version
-	if version == "" {
-		version = beatVersion
+	if esVersion == "" {
+		esVersion = beatVersion
 	}
 
-	esVersion, err := NewVersion(version)
+	esV, err := NewVersion(esVersion)
 	if err != nil {
 		return nil, err
 	}
 
-	fields := Fields{}
+	return &Template{
+		index:       index,
+		beatVersion: *bV,
+		esVersion:   *esV,
+	}, nil
 
-	for _, file := range files {
-		f, err := loadYaml(file)
-		if err != nil {
-			return nil, err
-		}
-		fields = append(fields, f...)
+}
+
+// Load the given input and generates the input based on it
+func (t *Template) Load(file string) (common.MapStr, error) {
+
+	fields, err := loadYaml(file)
+	if err != nil {
+		return nil, err
 	}
 
 	// Start processing at the root
-	properties := fields.process("", *esVersion)
-
-	indexPattern := fmt.Sprintf("%s-%s-*", beatName, beatVersion)
-	output := createTemplate(properties, beatVersion, *esVersion, indexPattern, dynamicTemplates)
+	properties := fields.process("", t.esVersion)
+	output := t.generate(properties, dynamicTemplates)
 
 	return output, nil
+}
 
+// GetName returns the name of the template which is {index}-{version}
+func (t *Template) GetName() string {
+	return fmt.Sprintf("%s-%s", t.index, t.beatVersion.String())
 }
 
-// createTemplate creates the full template
+// generate generates the full template
 // The default values are taken from the default variable.
-func createTemplate(properties common.MapStr, version string, esVersion Version, indexPattern string, dynamicTemplates []common.MapStr) common.MapStr {
+func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr {
 
 	// Add base dynamic template
 	var dynamicTemplateBase = common.MapStr{
@@ -67,7 +83,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
 		},
 	}
 
-	if esVersion.IsMajor(2) {
+	if t.esVersion.IsMajor(2) {
 		dynamicTemplateBase.Put("strings_as_keyword.mapping.type", "string")
 		dynamicTemplateBase.Put("strings_as_keyword.mapping.index", "not_analyzed")
 	}
@@ -79,7 +95,7 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
 		"mappings": common.MapStr{
 			"_default_": common.MapStr{
 				"_meta": common.MapStr{
-					"version": version,
+					"version": t.beatVersion.String(),
 				},
 				"date_detection":    defaultDateDetection,
 				"dynamic_templates": dynamicTemplates,
@@ -90,10 +106,10 @@ func createTemplate(properties common.MapStr, version string, esVersion Version,
 		"settings": common.MapStr{
 			"index.refresh_interval": "5s",
 		},
-		"template": indexPattern,
+		"template": t.GetName() + "-*",
 	}
 
-	if esVersion.IsMajor(2) {
+	if t.esVersion.IsMajor(2) {
 		basicStructure.Put("mappings._default_._all.norms.enabled", false)
 	} else {
 		// Metricbeat exceeds the default of 1000 fields