use many config files

cfg/config.go

@@ -13,7 +13,8 @@ import (
 
 	"github.com/bitly/go-simplejson"
 	"github.com/ozontech/file.d/logger"
-	"sigs.k8s.io/yaml"
+	"gopkg.in/yaml.v2"
+	k8s_yaml "sigs.k8s.io/yaml"
 )
 
 const trueValue = "true"
@@ -77,22 +78,37 @@ func NewConfig() *Config {
 	}
 }
 
-func NewConfigFromFile(path string) *Config {
-	logger.Infof("reading config %q", path)
-	yamlContents, err := os.ReadFile(path)
+func NewConfigFromFile(paths []string) *Config {
+	mergedConfig := make(map[interface{}]interface{})
+
+	for _, path := range paths {
+		logger.Infof("reading config %q", path)
+		yamlContents, err := os.ReadFile(path)
+		if err != nil {
+			logger.Fatalf("can't read config file %q: %s", path, err)
+		}
+		var currentConfig map[interface{}]interface{}
+		if err := yaml.Unmarshal(yamlContents, &currentConfig); err != nil {
+			logger.Fatalf("can't parse config file yaml %q: %s", path, err)
+		}
+
+		mergedConfig = mergeYAMLs(mergedConfig, currentConfig)
+	}
+
+	mergedYAML, err := yaml.Marshal(mergedConfig)
 	if err != nil {
-		logger.Fatalf("can't read config file %q: %s", path, err)
+		logger.Fatalf("can't marshal merged config to YAML: %s", err)
 	}
 
-	jsonContents, err := yaml.YAMLToJSON(yamlContents)
+	jsonContents, err := k8s_yaml.YAMLToJSON(mergedYAML)
 	if err != nil {
-		logger.Infof("config content:\n%s", logger.Numerate(string(yamlContents)))
-		logger.Fatalf("can't parse config file yaml %q: %s", path, err.Error())
+		logger.Infof("config content:\n%s", logger.Numerate(string(mergedYAML)))
+		logger.Fatalf("can't parse config file yaml %q: %s", paths, err.Error())
 	}
 
 	object, err := simplejson.NewJson(jsonContents)
 	if err != nil {
-		logger.Fatalf("can't convert config to json %q: %s", path, err.Error())
+		logger.Fatalf("can't convert config to json %q: %s", paths, err.Error())
 	}
 
 	err = applyEnvs(object)
@@ -637,3 +653,22 @@ func CompileRegex(s string) (*regexp.Regexp, error) {
 
 	return regexp.Compile(s[1 : len(s)-1])
 }
+
+func mergeYAMLs(a, b map[interface{}]interface{}) map[interface{}]interface{} {
+	merged := make(map[interface{}]interface{})
+	for k, v := range a {
+		merged[k] = v
+	}
+	for k, v := range b {
+		if existingValue, exists := merged[k]; exists {
+			if existingMap, ok := existingValue.(map[interface{}]interface{}); ok {
+				if newMap, ok := v.(map[interface{}]interface{}); ok {
+					merged[k] = mergeYAMLs(existingMap, newMap)
+					continue
+				}
+			}
+		}
+		merged[k] = v
+	}
+	return merged
+}

cfg/config_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func NewTestConfig(name string) *Config {
-	return NewConfigFromFile("../testdata/config/" + name)
+	return NewConfigFromFile([]string{"../testdata/config/" + name})
 }
 
 func TestSimple(t *testing.T) {

cmd/file.d/file.d.go

@@ -65,7 +65,7 @@ var (
 	fileD *fd.FileD
 	exit  = make(chan bool)
 
-	config        = kingpin.Flag("config", `Config file name`).Required().ExistingFile()
+	config        = kingpin.Flag("config", `Config file name (to add a config file you can repeat the argument)`).Required().ExistingFiles()
 	http          = kingpin.Flag("http", `HTTP listen addr eg. ":9000", "off" to disable`).Default(":9000").String()
 	memLimitRatio = kingpin.Flag(
 		"mem-limit-ratio",

cmd/file.d/file.d_test.go

@@ -85,7 +85,7 @@ func TestEndToEnd(t *testing.T) {
 	filesDir := t.TempDir()
 	offsetsDir := t.TempDir()
 
-	config := cfg.NewConfigFromFile(configFilename)
+	config := cfg.NewConfigFromFile([]string{configFilename})
 	input := config.Pipelines["test"].Raw.Get("input")
 	input.Set("watching_dir", filesDir)
 	input.Set("offsets_file", filepath.Join(offsetsDir, "offsets.yaml"))

go.mod

@@ -48,6 +48,7 @@ require (
 	go.uber.org/zap v1.25.0
 	golang.org/x/net v0.21.0
 	google.golang.org/protobuf v1.33.1-0.20240408130810-98873a205002
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.27.4
 	k8s.io/apimachinery v0.27.4
@@ -149,7 +150,6 @@ require (
 	google.golang.org/appengine v1.6.7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect