Update Filebeat hints to also support modules (#6899)

* Unify builder hints docker info under `container`

Before this change `kubernetes` provider was using `container` while
`docker` provider was using `docker.container`. This change unifies
container info under `container`, this opens the door to other runtimes
in the future.

* Add support for module hints in Filebeat

filebeat/autodiscover/builder/hints/config.go

@@ -3,22 +3,22 @@ package hints
 import "github.com/elastic/beats/libbeat/common"
 
 type config struct {
-	Key    string           `config:"key"`
-	Config []*common.Config `config:"config"`
+	Key    string         `config:"key"`
+	Config *common.Config `config:"config"`
 }
 
 func defaultConfig() config {
 	rawCfg := map[string]interface{}{
 		"type": "docker",
 		"containers": map[string]interface{}{
 			"ids": []string{
-				"${data.docker.container.id}",
+				"${data.container.id}",
 			},
 		},
 	}
 	cfg, _ := common.NewConfigFrom(rawCfg)
 	return config{
 		Key:    "logs",
-		Config: []*common.Config{cfg},
+		Config: cfg,
 	}
 }

filebeat/autodiscover/builder/hints/logs.go

@@ -2,7 +2,9 @@ package hints
 
 import (
 	"fmt"
+	"regexp"
 
+	"github.com/elastic/beats/filebeat/fileset"
 	"github.com/elastic/beats/libbeat/autodiscover"
 	"github.com/elastic/beats/libbeat/autodiscover/builder"
 	"github.com/elastic/beats/libbeat/autodiscover/template"
@@ -22,9 +24,13 @@ const (
 	excludeLines = "exclude_lines"
 )
 
+// validModuleNames to sanitize user input
+var validModuleNames = regexp.MustCompile("[^a-zA-Z0-9]+")
+
 type logHints struct {
-	Key    string
-	Config []*common.Config
+	Key      string
+	Config   *common.Config
+	Registry *fileset.ModuleRegistry
 }
 
 // NewLogHints builds a log hints builder
@@ -37,16 +43,21 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
 		return nil, fmt.Errorf("unable to unpack hints config due to error: %v", err)
 	}
 
-	return &logHints{config.Key, config.Config}, nil
+	moduleRegistry, err := fileset.NewModuleRegistry([]*common.Config{}, "", false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &logHints{config.Key, config.Config, moduleRegistry}, nil
 }
 
 // Create config based on input hints in the bus event
 func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
-	var config []*common.Config
-
+	// Clone original config
+	config, _ := common.NewConfigFrom(l.Config)
 	host, _ := event["host"].(string)
 	if host == "" {
-		return config
+		return []*common.Config{}
 	}
 
 	var hints common.MapStr
@@ -56,11 +67,9 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 	}
 
 	if builder.IsNoOp(hints, l.Key) == true {
-		return config
+		return []*common.Config{config}
 	}
 
-	//TODO: Add module support
-
 	tempCfg := common.MapStr{}
 	mline := l.getMultiline(hints)
 	if len(mline) != 0 {
@@ -74,18 +83,34 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 	}
 
 	// Merge config template with the configs from the annotations
-	for _, c := range l.Config {
-		if err := c.Merge(tempCfg); err != nil {
-			logp.Debug("hints.builder", "config merge failed with error: %v", err)
-		} else {
-			logp.Debug("hints.builder", "generated config %v", *c)
-			config = append(config, c)
+	if err := config.Merge(tempCfg); err != nil {
+		logp.Debug("hints.builder", "config merge failed with error: %v", err)
+		return []*common.Config{config}
+	}
+
+	module := l.getModule(hints)
+	if module != "" {
+		moduleConf := map[string]interface{}{
+			"module": module,
 		}
+
+		filesets := l.getFilesets(hints, module)
+		for fileset, conf := range filesets {
+			filesetConf, _ := common.NewConfigFrom(config)
+			filesetConf.SetString("containers.stream", -1, conf.Stream)
+
+			moduleConf[fileset+".enabled"] = conf.Enabled
+			moduleConf[fileset+".input"] = filesetConf
+
+			logp.Debug("hints.builder", "generated config %+v", moduleConf)
+		}
+		config, _ = common.NewConfigFrom(moduleConf)
 	}
 
+	logp.Debug("hints.builder", "generated config %+v", config)
+
 	// Apply information in event to the template to generate the final config
-	config = template.ApplyConfigTemplate(event, config)
-	return config
+	return template.ApplyConfigTemplate(event, []*common.Config{config})
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {
@@ -99,3 +124,60 @@ func (l *logHints) getIncludeLines(hints common.MapStr) []string {
 func (l *logHints) getExcludeLines(hints common.MapStr) []string {
 	return builder.GetHintAsList(hints, l.Key, excludeLines)
 }
+
+func (l *logHints) getModule(hints common.MapStr) string {
+	module := builder.GetHintString(hints, l.Key, "module")
+	// for security, strip module name
+	return validModuleNames.ReplaceAllString(module, "")
+}
+
+type filesetConfig struct {
+	Enabled bool
+	Stream  string
+}
+
+// Return a map containing filesets -> enabled & stream (stdout, stderr, all)
+func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*filesetConfig {
+	var configured bool
+	filesets := make(map[string]*filesetConfig)
+
+	moduleFilesets, err := l.Registry.ModuleFilesets(module)
+	if err != nil {
+		logp.Err("Error retrieving module filesets", err)
+		return nil
+	}
+
+	for _, fileset := range moduleFilesets {
+		filesets[fileset] = &filesetConfig{Enabled: false, Stream: "all"}
+	}
+
+	// If a single fileset is given, pass all streams to it
+	fileset := builder.GetHintString(hints, l.Key, "fileset")
+	if fileset != "" {
+		if conf, ok := filesets[fileset]; ok {
+			conf.Enabled = true
+			configured = true
+		}
+	}
+
+	// If fileset is defined per stream, return all of them
+	for _, stream := range []string{"all", "stdout", "stderr"} {
+		fileset := builder.GetHintString(hints, l.Key, "fileset."+stream)
+		if fileset != "" {
+			if conf, ok := filesets[fileset]; ok {
+				conf.Enabled = true
+				conf.Stream = stream
+				configured = true
+			}
+		}
+	}
+
+	// No fileseat defined, return defaults for the module, all streams to all filesets
+	if !configured {
+		for _, conf := range filesets {
+			conf.Enabled = true
+		}
+	}
+
+	return filesets
+}