Combine fields.yml properties when they are defined in different so…

…urces

We define fields with the same root key across several files, this
change ensures all of them are combined in the final index template.

Fixes elastic#5075

CHANGELOG.asciidoc

@@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Register kubernetes `field_format` matcher and remove logger in `Encode` API {pull}4888[4888]
 - Add support for `initContainers` in `add_kubernetes_metadata` processor. {issue}4825[4825]
 - Fix the `/usr/bin/beatname` script to accept `-d "*"` as a parameter. {issue}5040[5040]
+- Combine `fields.yml` properties when they are defined in different sources. {issue}5075[5075]
 
 *Auditbeat*
 

libbeat/template/field.go

@@ -131,7 +131,9 @@ func (f *Field) text() common.MapStr {
 	}
 
 	if len(f.MultiFields) > 0 {
-		properties["fields"] = f.MultiFields.process("", f.esVersion)
+		fields := common.MapStr{}
+		f.MultiFields.process("", f.esVersion, fields)
+		properties["fields"] = fields
 	}
 
 	return properties

libbeat/template/field_test.go

@@ -3,9 +3,10 @@ package template
 import (
 	"testing"
 
+	"github.com/elastic/beats/libbeat/common"
+
 	"github.com/stretchr/testify/assert"
 
-	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/go-ucfg/yaml"
 )
 

libbeat/template/fields.go

@@ -1,6 +1,7 @@
 package template
 
 import (
+	"errors"
 	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -12,9 +13,7 @@ var (
 
 type Fields []Field
 
-func (f Fields) process(path string, esVersion common.Version) common.MapStr {
-	output := common.MapStr{}
-
+func (f Fields) process(path string, esVersion common.Version, output common.MapStr) error {
 	for _, field := range f {
 
 		var mapping common.MapStr
@@ -54,7 +53,25 @@ func (f Fields) process(path string, esVersion common.Version) common.MapStr {
 			if field.Dynamic.value != nil {
 				mapping["dynamic"] = field.Dynamic.value
 			}
-			mapping["properties"] = field.Fields.process(newPath, esVersion)
+
+			// Combine properties with previous field definitions (if any)
+			properties := common.MapStr{}
+			key := generateKey(field.Name) + ".properties"
+			currentProperties, err := output.GetValue(key)
+			if err == nil {
+				var ok bool
+				properties, ok = currentProperties.(common.MapStr)
+				if !ok {
+					// This should never happen
+					return errors.New(key + " is expected to be a MapStr")
+				}
+			}
+
+			if err := field.Fields.process(newPath, esVersion, properties); err != nil {
+				return err
+			}
+			mapping["properties"] = properties
+
 		default:
 			mapping = field.other()
 		}
@@ -64,7 +81,7 @@ func (f Fields) process(path string, esVersion common.Version) common.MapStr {
 		}
 	}
 
-	return output
+	return nil
 }
 
 // HasKey checks if inside fields the given key exists

libbeat/template/fields_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/common"
 )
 
 func TestHasKey(t *testing.T) {
@@ -57,3 +59,52 @@ func TestHasKey(t *testing.T) {
 		assert.Equal(t, test.result, test.fields.HasKey(test.key))
 	}
 }
+
+func TestPropertiesCombine(t *testing.T) {
+	// Test common fields are combined even if they come from different objects
+	fields := Fields{
+		Field{
+			Name: "test",
+			Type: "group",
+			Fields: Fields{
+				Field{
+					Name: "one",
+					Type: "text",
+				},
+			},
+		},
+		Field{
+			Name: "test",
+			Type: "group",
+			Fields: Fields{
+				Field{
+					Name: "two",
+					Type: "text",
+				},
+			},
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("6.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = fields.process("", *version, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	v1, err := output.GetValue("test.properties.one")
+	if err != nil {
+		t.Fatal(err)
+	}
+	v2, err := output.GetValue("test.properties.two")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, v1, common.MapStr{"type": "text", "norms": false})
+	assert.Equal(t, v2, common.MapStr{"type": "text", "norms": false})
+}

libbeat/template/template.go

@@ -99,7 +99,10 @@ func (t *Template) Load(file string) (common.MapStr, error) {
 	}
 
 	// Start processing at the root
-	properties := fields.process("", t.esVersion)
+	properties := common.MapStr{}
+	if err := fields.process("", t.esVersion, properties); err != nil {
+		return nil, err
+	}
 	output := t.generate(properties, dynamicTemplates)
 
 	return output, nil