Unify fields generated file across all beats (#3677)

Each beat has its own way of generating the fields.yml file. Some have `_meta/fields.yml`, others `_meta/fields.generated.yml`. All scripts that used the fields.yml file had to do checks for these files and include the `libbeat/_meta/fields.generate.yml` file to get all data in. Now all beats will create `_meta/fields.generated.yml`. In case it does not exist, it is just a copy of `_meta/fields.yml` and will be automatically generated. In addition the file `_meta/fields.full.generated.yml` was introduced, which also contains the content from the `libbeat/_meta/fields.generate.yml`. This file can be used in most scripts to get all fields. The scripts were changed / simplified accordingly.

This simplifies the implementation of dynamic template loading as only 1 file has to be taken into account.

Part of #3654

libbeat/scripts/Makefile

@@ -244,14 +244,20 @@ update: python-env collect
 	# make sure generated docs in libbeat are up-to-date
 	$(MAKE) -C ${ES_BEATS}/libbeat fields
 
+	# Generate full fields for each beat. Create fields.generated.yml in case it does not exist yet.
+	# Most beats have their own way of generating it.
+	test -s  _meta/fields.generated.yml || cp _meta/fields.yml _meta/fields.generated.yml
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml _meta/fields.generated.yml > _meta/fields.full.generated.yml
+	# Revert changes to libbeat fields as it otherwise contains duplicates in full
+	cat ${ES_BEATS}/libbeat/_meta/fields.generated.yml > ${ES_BEATS}/libbeat/_meta/fields.full.generated.yml
+
 	# Update docs
 	mkdir -p docs
 	. ${PYTHON_ENV}/bin/activate && python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_NAME} ${ES_BEATS}
 
 	# Generate index templates
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
-
-	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${PWD}/${ES_BEATS}/libbeat/_meta/fields.generated.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.yml ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.generated.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
+	go run ${ES_BEATS}/dev-tools/cmd/index_template/index_template.go -es.version 2.0.0 -beat.name ${BEAT_NAME} -output ${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME}.template-es2x.json ${BEAT_GOPATH}/src/${BEAT_PATH}/_meta/fields.full.generated.yml
 
 	# Generate index-pattern
 	echo "Generate index pattern"

libbeat/scripts/generate_fields_docs.py

@@ -111,20 +111,12 @@ def fields_to_asciidoc(input, output, beat):
     beat_name = args.beatname
     es_beats = args.es_beats
 
-    fields_yml = beat_path + "/_meta/fields.generated.yml"
-
-    # Not all beats have a fields.generated.yml. Fall back to fields.yml
-    if not os.path.isfile(fields_yml):
-        fields_yml = beat_path + "/_meta/fields.yml"
+    fields_yml = beat_path + "/_meta/fields.full.generated.yml"
 
     # Read fields.yml
     with open(fields_yml) as f:
         fields = f.read()
 
-    # Prepends beat fields from libbeat
-    with open(es_beats + "/libbeat/_meta/fields.generated.yml") as f:
-        fields = f.read() + fields
-
     output = open(beat_path + "/docs/fields.asciidoc", 'w')
 
     try:

libbeat/scripts/generate_index_pattern.py

@@ -139,21 +139,12 @@ def get_index_pattern_name(index):
 
     args = parser.parse_args()
 
-    fields_yml = args.beat + "/_meta/fields.generated.yml"
-
-    # Not all beats have a fields.generated.yml. Fall back to fields.yml
-    if not os.path.isfile(fields_yml):
-        fields_yml = args.beat + "/_meta/fields.yml"
+    fields_yml = args.beat + "/_meta/fields.full.generated.yml"
 
     # generate the index-pattern content
     with open(fields_yml, 'r') as f:
         fields = f.read()
 
-        if os.path.basename(args.beat) != "libbeat":
-            # Prepend beat fields from libbeat
-            with open(args.libbeat + "/_meta/fields.generated.yml") as f:
-                fields = f.read() + fields
-
         # with open(target, 'w') as output:
         output = fields_to_index_pattern(args, fields)