[Filebeat] Refactor autodiscover default input settings handling (#12193

)

Autodiscover hints default config (unreleased) was done at the hints
level, while it only makes sense for certain configs builder (logs).

This change moves this logic to the `logs` builder and improves how
users can define default settings for logs, ie:

Enable hints but disable log retrieval by default:

```
filebeat.autodiscover:
  providers:
    - type: docker # or kubernetes
      hints.enabled: true
      hints.default_config.enabled: false
```

Only containers with the `co.elastic.logs/enabled: true` annotation
will be retrieved.

```
filebeat.autodiscover:
  providers:
    - type: kubernetes # or docker
      hints.enabled: true
      hints.default_config:
        type: container
        paths:
          - /var/log/containers/*${data.container.id}.log
```

Logs are read by default for all containers, using the given input
settings. Containers with `co.elastic.logs/enabled: false` annotation
will be ignored.

CHANGELOG.next.asciidoc

@@ -175,7 +175,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add option to configure docker input with paths {pull}10687[10687]
 - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877]
 - Set `event.category: network_traffic` for Suricata. {pull}10882[10882]
-- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911]
+- Allow custom default settings with autodiscover (for example, use of CRI paths for logs). {pull}12193[12193]
+- Allow to disable hints based autodiscover default behavior (fetching all logs). {pull}12193[12193]
 - Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
 - Add the `network.community_id` flow identifier to field to the IPTables, Suricata, and Zeek modules. {pull}11005[11005]
 - New Filebeat coredns module to ingest coredns logs. It supports both native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200]

filebeat/autodiscover/builder/hints/config.go

@@ -20,12 +20,12 @@ package hints
 import "github.com/elastic/beats/libbeat/common"
 
 type config struct {
-	Key    string         `config:"key"`
-	Config *common.Config `config:"config"`
+	Key           string         `config:"key"`
+	DefaultConfig *common.Config `config:"default_config"`
 }
 
 func defaultConfig() config {
-	rawCfg := map[string]interface{}{
+	defaultCfgRaw := map[string]interface{}{
 		"type": "container",
 		"containers": map[string]interface{}{
 			"paths": []string{
@@ -35,10 +35,10 @@ func defaultConfig() config {
 			},
 		},
 	}
-	cfg, _ := common.NewConfigFrom(rawCfg)
+	defaultCfg, _ := common.NewConfigFrom(defaultCfgRaw)
 	return config{
-		Key:    "logs",
-		Config: cfg,
+		Key:           "logs",
+		DefaultConfig: defaultCfg,
 	}
 }
 
@@ -52,8 +52,8 @@ func (c *config) Unpack(from *common.Config) error {
 		return err
 	}
 
-	if config, err := from.Child("config", -1); err == nil {
-		c.Config = config
+	if config, err := from.Child("default_config", -1); err == nil {
+		c.DefaultConfig = config
 	}
 
 	c.Key = tmpConfig.Key

filebeat/autodiscover/builder/hints/logs.go

@@ -27,7 +27,6 @@ import (
 	"github.com/elastic/beats/libbeat/autodiscover/template"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/bus"
-	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
@@ -46,14 +45,12 @@ const (
 var validModuleNames = regexp.MustCompile("[^a-zA-Z0-9\\_\\-]+")
 
 type logHints struct {
-	Key      string
-	Config   *common.Config
-	Registry *fileset.ModuleRegistry
+	config   *config
+	registry *fileset.ModuleRegistry
 }
 
 // NewLogHints builds a log hints builder
 func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
-	cfgwarn.Beta("The hints builder is beta")
 	config := defaultConfig()
 	err := cfg.Unpack(&config)
 
@@ -66,30 +63,38 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
 		return nil, err
 	}
 
-	return &logHints{config.Key, config.Config, moduleRegistry}, nil
+	return &logHints{&config, moduleRegistry}, nil
 }
 
 // Create config based on input hints in the bus event
 func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
-	// Clone original config
-	config, _ := common.NewConfigFrom(l.Config)
-	host, _ := event["host"].(string)
-	if host == "" {
-		return []*common.Config{}
-	}
-
 	var hints common.MapStr
 	hIface, ok := event["hints"]
 	if ok {
 		hints, _ = hIface.(common.MapStr)
 	}
 
-	if builder.IsNoOp(hints, l.Key) {
-		logp.Debug("hints.builder", "disabled config in event: %+v", event)
+	inputConfig := l.getInputs(hints)
+
+	// If default config is disabled return nothing unless it's explicty enabled
+	if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) {
+		logp.Debug("hints.builder", "default config is disabled: %+v", event)
+		return []*common.Config{}
+	}
+
+	// If explicty disabled, return nothing
+	if builder.IsDisabled(hints, l.config.Key) {
+		logp.Debug("hints.builder", "logs disabled by hint: %+v", event)
+		return []*common.Config{}
+	}
+
+	// Clone original config
+	config, _ := common.NewConfigFrom(l.config.DefaultConfig)
+	host, _ := event["host"].(string)
+	if host == "" {
 		return []*common.Config{}
 	}
 
-	inputConfig := l.getInputs(hints)
 	if inputConfig != nil {
 		configs := []*common.Config{}
 		for _, cfg := range inputConfig {
@@ -149,29 +154,29 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {
-	return builder.GetHintMapStr(hints, l.Key, multiline)
+	return builder.GetHintMapStr(hints, l.config.Key, multiline)
 }
 
 func (l *logHints) getIncludeLines(hints common.MapStr) []string {
-	return builder.GetHintAsList(hints, l.Key, includeLines)
+	return builder.GetHintAsList(hints, l.config.Key, includeLines)
 }
 
 func (l *logHints) getExcludeLines(hints common.MapStr) []string {
-	return builder.GetHintAsList(hints, l.Key, excludeLines)
+	return builder.GetHintAsList(hints, l.config.Key, excludeLines)
 }
 
 func (l *logHints) getModule(hints common.MapStr) string {
-	module := builder.GetHintString(hints, l.Key, "module")
+	module := builder.GetHintString(hints, l.config.Key, "module")
 	// for security, strip module name
 	return validModuleNames.ReplaceAllString(module, "")
 }
 
 func (l *logHints) getInputs(hints common.MapStr) []common.MapStr {
-	return builder.GetHintAsConfigs(hints, l.Key)
+	return builder.GetHintAsConfigs(hints, l.config.Key)
 }
 
 func (l *logHints) getProcessors(hints common.MapStr) []common.MapStr {
-	return builder.GetProcessors(hints, l.Key)
+	return builder.GetProcessors(hints, l.config.Key)
 }
 
 type filesetConfig struct {
@@ -184,7 +189,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 	var configured bool
 	filesets := make(map[string]*filesetConfig)
 
-	moduleFilesets, err := l.Registry.ModuleFilesets(module)
+	moduleFilesets, err := l.registry.ModuleFilesets(module)
 	if err != nil {
 		logp.Err("Error retrieving module filesets: %+v", err)
 		return nil
@@ -195,7 +200,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 	}
 
 	// If a single fileset is given, pass all streams to it
-	fileset := builder.GetHintString(hints, l.Key, "fileset")
+	fileset := builder.GetHintString(hints, l.config.Key, "fileset")
 	if fileset != "" {
 		if conf, ok := filesets[fileset]; ok {
 			conf.Enabled = true
@@ -205,7 +210,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 
 	// If fileset is defined per stream, return all of them
 	for _, stream := range []string{"all", "stdout", "stderr"} {
-		fileset := builder.GetHintString(hints, l.Key, "fileset."+stream)
+		fileset := builder.GetHintString(hints, l.config.Key, "fileset."+stream)
 		if fileset != "" {
 			if conf, ok := filesets[fileset]; ok {
 				conf.Enabled = true

filebeat/autodiscover/builder/hints/logs_test.go

@@ -377,7 +377,7 @@ func TestGenerateHints(t *testing.T) {
 
 	for _, test := range tests {
 		cfg, _ := common.NewConfigFrom(map[string]interface{}{
-			"config": map[string]interface{}{
+			"default_config": map[string]interface{}{
 				"type": "docker",
 				"containers": map[string]interface{}{
 					"ids": []string{
@@ -609,7 +609,7 @@ func TestGenerateHintsWithPaths(t *testing.T) {
 
 	for _, test := range tests {
 		cfg, _ := common.NewConfigFrom(map[string]interface{}{
-			"config": map[string]interface{}{
+			"default_config": map[string]interface{}{
 				"type": "docker",
 				"containers": map[string]interface{}{
 					"paths": []string{

filebeat/docs/autodiscover-hints.asciidoc

@@ -2,14 +2,16 @@
 hints in Kubernetes Pod annotations or Docker labels that have the prefix `co.elastic.logs`. As soon as
 the container starts, {beatname_uc} will check if it contains any hints and launch the proper config for
 it. Hints tell {beatname_uc} how to get logs for the given container. By default logs will be retrieved
-from the container using the `docker` input. You can use hints to modify this behavior. This is the full
+from the container using the `container` input. You can use hints to modify this behavior. This is the full
 list of supported hints:
 
 [float]
-===== `co.elastic.logs/disable`
+===== `co.elastic.logs/enabled`
 
-Filebeat gets logs from all containers by default, you can set this hint to `true` to ignore
-the output of the container. Filebeat won't read or send logs from it.
+Filebeat gets logs from all containers by default, you can set this hint to `false` to ignore
+the output of the container. Filebeat won't read or send logs from it. If default config is
+disabled, you can use this annotation to enable log retrieval only for containers with this
+set to `true`.
 
 [float]
 ===== `co.elastic.logs/multiline.*`
@@ -94,23 +96,30 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
-Autodiscover provides a way to control whether log harvesting is by default disabled for the pods/containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+You can configure the default config that will be launched when a new container is seen, like this:
 
 ["source","yaml",subs="attributes"]
 -------------------------------------------------------------------------------------
 filebeat.autodiscover:
   providers:
     - type: kubernetes
       hints.enabled: true
-      default.disable: true
+      hints.default_config:
+        type: container
+        paths:
+          /var/log/container/*-${container.id}.log  # CRI path
 -------------------------------------------------------------------------------------
 
-Then, for the pods/containers that log harvesting should be enabled, you can annotate with hint:
+You can also disable default settings entirely, so only Pods annotated like `co.elastic.logs/enabled: true`
+will be retrieved:
 
 ["source","yaml",subs="attributes"]
 -------------------------------------------------------------------------------------
-annotations:
-  co.elastic.logs/disable: false
+filebeat.autodiscover:
+  providers:
+    - type: kubernetes
+      hints.enabled: true
+      hints.default_config.enabled: false
 -------------------------------------------------------------------------------------
 
 You can annotate Kubernetes Pods with useful info to spin up {beatname_uc} inputs or modules:
@@ -156,24 +165,30 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
-Autodiscover provides a way to control whether log harvesting is by default disabled for the 
-containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+You can configure the default config that will be launched when a new container is seen, like this:
 
 ["source","yaml",subs="attributes"]
 -------------------------------------------------------------------------------------
 filebeat.autodiscover:
   providers:
     - type: docker
       hints.enabled: true
-      default.disable: true
+      hints.default_config:
+        type: container
+        paths:
+          /var/log/container/*-${container.id}.log  # CRI path
 -------------------------------------------------------------------------------------
 
-Then, for the containers that log harvesting should be enabled, you can label Docker containers with:
+You can also disable default settings entirely, so only containers labeled with `co.elastic.logs/enabled: true`
+will be retrieved:
 
 ["source","yaml",subs="attributes"]
 -------------------------------------------------------------------------------------
-annotations:
-  co.elastic.logs/disable: false
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      hints.default_config.enabled: false
 -------------------------------------------------------------------------------------
 
 You can label Docker containers with useful info to spin up {beatname_uc} inputs, for example:

filebeat/input/docker/input.go

@@ -68,7 +68,7 @@ func NewInput(
 	}
 
 	if len(ids) == 0 {
-		return nil, errors.New("Docker input requires at least one entry under 'containers.ids''")
+		return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'")
 	}
 
 	for idx, containerID := range ids {

filebeat/tests/system/test_autodiscover.py

@@ -56,3 +56,49 @@ def test_docker(self):
         assert 'name' in output[0]['container']
 
         self.assert_fields_are_documented(output[0])
+
+    @unittest.skipIf(not INTEGRATION_TESTS or
+                     os.getenv("TESTING_ENVIRONMENT") == "2x",
+                     "integration test not available on 2.x")
+    def test_default_settings(self):
+        """
+        Test docker autodiscover default config settings
+        """
+        import docker
+        docker_client = docker.from_env()
+
+        self.render_config_template(
+            inputs=False,
+            autodiscover={
+                'docker': {
+                    'cleanup_timeout': '0s',
+                    'hints.enabled': 'true',
+                    'hints.default_config': '''
+                      type: log
+                      paths:
+                        - %s/${data.container.image}.log
+                    ''' % self.working_dir,
+                },
+            },
+        )
+
+        with open(os.path.join(self.working_dir, 'busybox.log'), 'wb') as f:
+            f.write('Busybox output 1\n')
+
+        proc = self.start_beat()
+        docker_client.images.pull('busybox')
+        docker_client.containers.run('busybox', 'sleep 1')
+
+        self.wait_until(lambda: self.log_contains('Starting runner: input'))
+        self.wait_until(lambda: self.log_contains('Stopping runner: input'))
+
+        output = self.read_output_json()
+        proc.check_kill_and_wait()
+
+        # Check metadata is added
+        assert output[0]['message'] == 'Busybox output 1'
+        assert output[0]['container']['image']['name'] == 'busybox'
+        assert output[0]['docker']['container']['labels'] == {}
+        assert 'name' in output[0]['container']
+
+        self.assert_fields_are_documented(output[0])

libbeat/autodiscover/builder.go

@@ -99,21 +99,18 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config {
 	return configs
 }
 
-// NewBuilders instances the given list of builders. If hintsEnabled is true it will
-// just enable the hints builder
-func NewBuilders(bConfigs []*common.Config, hintsEnabled bool) (Builders, error) {
+// NewBuilders instances the given list of builders. hintsCfg holds `hints` settings
+// for simplified mode (single 'hints' builder)
+func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) {
 	var builders Builders
-	if hintsEnabled {
+	if hintsCfg.Enabled() {
 		if len(bConfigs) > 0 {
 			return nil, errors.New("hints.enabled is incompatible with manually defining builders")
 		}
 
-		hints, err := common.NewConfigFrom(map[string]string{"type": "hints"})
-		if err != nil {
-			return nil, err
-		}
-
-		bConfigs = append(bConfigs, hints)
+		// pass rest of hints settings to the builder
+		hintsCfg.SetString("type", -1, "hints")
+		bConfigs = append(bConfigs, hintsCfg)
 	}
 
 	for _, bcfg := range bConfigs {