Add configuration knob for auto-discover default.disable

CHANGELOG.next.asciidoc

@@ -284,6 +284,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added support for ingesting structured Elasticsearch server logs {pull}10428[10428]
 - Populate more ECS fields in the Suricata module. {pull}10006[10006]
 - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736]
+- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811]
 
 *Heartbeat*
 

filebeat/autodiscover/builder/hints/logs_test.go

@@ -47,6 +47,18 @@ func TestGenerateHints(t *testing.T) {
 			len:    0,
 			result: common.MapStr{},
 		},
+		{
+			msg: "Hints with logs.disable should return nothing",
+			event: bus.Event{
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"disable": "true",
+					},
+				},
+			},
+			len:    0,
+			result: common.MapStr{},
+		},
 		{
 			msg: "Empty event hints should return default config",
 			event: bus.Event{

filebeat/docs/modules/kubernetes.asciidoc

@@ -0,0 +1,35 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-kubernetes]]
+[role="xpack"]
+
+:modulename: kubernetes
+:has-dashboards: true
+
+== Kubernetes Module
+
+This is a filebeat module for Kubernetes. It ingests logs for Kubernetes pods and containers, specifically coredns and envoy.
+
+[float]
+=== Compatibility
+
+This module has been developed against Kubernetes v1.13.x, but is expected to work
+with other versions of Kubernetes.
+
+[float]
+=== Example dashboard
+
+This module comes with a sample dashboard.
+
+[role="screenshot"]
+image::./images/Filebeat-Kubernetes-Dashboard.png[]
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-kubernetes,exported fields>> section.
+

libbeat/autodiscover/builder/helper.go

@@ -155,7 +155,7 @@ func IsNoOp(hints common.MapStr, key string) bool {
 }
 
 // GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats.
-func GenerateHints(annotations common.MapStr, container, prefix string) common.MapStr {
+func GenerateHints(annotations common.MapStr, container, prefix string, defaultDisable bool) common.MapStr {
 	hints := common.MapStr{}
 	if rawEntries, err := annotations.GetValue(prefix); err == nil {
 		if entries, ok := rawEntries.(common.MapStr); ok {
@@ -195,5 +195,10 @@ func GenerateHints(annotations common.MapStr, container, prefix string) common.M
 		}
 	}
 
+	// Update hints: if .disabled annotation does not exist, set according to disabledByDefault flag
+	if _, err := hints.GetValue("logs.disable"); err != nil && defaultDisable {
+		hints.Put("logs.disable", "true")
+	}
+
 	return hints
 }

libbeat/autodiscover/builder/helper_test.go

@@ -27,13 +27,15 @@ import (
 
 func TestGenerateHints(t *testing.T) {
 	tests := []struct {
-		annotations map[string]string
-		result      common.MapStr
+		annotations    map[string]string
+		defaultDisable bool
+		result         common.MapStr
 	}{
 		// Empty annotations should return empty hints
 		{
-			annotations: map[string]string{},
-			result:      common.MapStr{},
+			annotations:    map[string]string{},
+			defaultDisable: false,
+			result:         common.MapStr{},
 		},
 
 		// Scenarios being tested:
@@ -50,6 +52,7 @@ func TestGenerateHints(t *testing.T) {
 				"co.elastic.metrics.foobar1/period": "15s",
 				"not.to.include":                    "true",
 			},
+			defaultDisable: false,
 			result: common.MapStr{
 				"logs": common.MapStr{
 					"multiline": common.MapStr{
@@ -62,13 +65,74 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		// Scenarios being tested:
+		// logs.disable must be generated when defaultDisable is set and annotations does not
+		// have co.elastic.logs/disable set to false.
+		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// metrics/module must be found in hints.metrics
+		// not.to.include must not be part of hints
+		// period is annotated at both container and pod level. Container level value must be in hints
+		{
+			annotations: map[string]string{
+				"co.elastic.logs/multiline.pattern": "^test",
+				"co.elastic.metrics/module":         "prometheus",
+				"co.elastic.metrics/period":         "10s",
+				"co.elastic.metrics.foobar/period":  "15s",
+				"co.elastic.metrics.foobar1/period": "15s",
+				"not.to.include":                    "true",
+			},
+			defaultDisable: true,
+			result: common.MapStr{
+				"logs": common.MapStr{
+					"multiline": common.MapStr{
+						"pattern": "^test",
+					},
+					"disable": "true",
+				},
+				"metrics": common.MapStr{
+					"module": "prometheus",
+					"period": "15s",
+				},
+			},
+		},
+		// Scenarios being tested:
+		// logs.disable must not be generated when defaultDisable is set, but annotations
+		// have co.elastic.logs/disable set to false.
+		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// metrics/module must be found in hints.metrics
+		// not.to.include must not be part of hints
+		// period is annotated at both container and pod level. Container level value must be in hints
+		{
+			annotations: map[string]string{
+				"co.elastic.logs/disable":           "false",
+				"co.elastic.logs/multiline.pattern": "^test",
+				"co.elastic.metrics/module":         "prometheus",
+				"co.elastic.metrics/period":         "10s",
+				"co.elastic.metrics.foobar/period":  "15s",
+				"co.elastic.metrics.foobar1/period": "15s",
+				"not.to.include":                    "true",
+			},
+			defaultDisable: true,
+			result: common.MapStr{
+				"logs": common.MapStr{
+					"multiline": common.MapStr{
+						"pattern": "^test",
+					},
+					"disable": "false",
+				},
+				"metrics": common.MapStr{
+					"module": "prometheus",
+					"period": "15s",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
 		annMap := common.MapStr{}
 		for k, v := range test.annotations {
 			annMap.Put(k, v)
 		}
-		assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic"), test.result)
+		assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic", test.defaultDisable), test.result)
 	}
 }

libbeat/autodiscover/providers/docker/config.go

@@ -25,13 +25,14 @@ import (
 
 // Config for docker autodiscover provider
 type Config struct {
-	Host         string                  `config:"host"`
-	TLS          *docker.TLSConfig       `config:"ssl"`
-	Prefix       string                  `config:"prefix"`
-	HintsEnabled bool                    `config:"hints.enabled"`
-	Builders     []*common.Config        `config:"builders"`
-	Appenders    []*common.Config        `config:"appenders"`
-	Templates    template.MapperSettings `config:"templates"`
+	Host           string                  `config:"host"`
+	TLS            *docker.TLSConfig       `config:"ssl"`
+	Prefix         string                  `config:"prefix"`
+	HintsEnabled   bool                    `config:"hints.enabled"`
+	DefaultDisable bool                    `config:"default.disable"`
+	Builders       []*common.Config        `config:"builders"`
+	Appenders      []*common.Config        `config:"appenders"`
+	Templates      template.MapperSettings `config:"templates"`
 }
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/docker/docker.go

@@ -212,7 +212,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 	if labels, err := dockerMeta.GetValue("labels"); err == nil {
-		hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix)
+		hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix, d.config.DefaultDisable)
 		e["hints"] = hints
 	}
 	return e

libbeat/autodiscover/providers/kubernetes/config.go

@@ -33,11 +33,12 @@ type Config struct {
 	SyncPeriod     time.Duration `config:"sync_period"`
 	CleanupTimeout time.Duration `config:"cleanup_timeout"`
 
-	Prefix       string                  `config:"prefix"`
-	HintsEnabled bool                    `config:"hints.enabled"`
-	Builders     []*common.Config        `config:"builders"`
-	Appenders    []*common.Config        `config:"appenders"`
-	Templates    template.MapperSettings `config:"templates"`
+	Prefix         string                  `config:"prefix"`
+	HintsEnabled   bool                    `config:"hints.enabled"`
+	DefaultDisable bool                    `config:"default.disable"`
+	Builders       []*common.Config        `config:"builders"`
+	Appenders      []*common.Config        `config:"appenders"`
+	Templates      template.MapperSettings `config:"templates"`
 }
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/kubernetes/kubernetes.go

@@ -18,7 +18,6 @@
 package kubernetes
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/gofrs/uuid"
@@ -145,16 +144,12 @@ func (p *Provider) emitEvents(pod *kubernetes.Pod, flag string, containers []*ku
 	containerstatuses []*kubernetes.PodContainerStatus) {
 	host := pod.Status.GetPodIP()
 
-	// If the container doesn't exist in the runtime or its network
-	// is not configured, it won't have an IP. Skip it as we cannot
-	// generate configs without host, and an update will arrive when
-	// the container is ready.
-	// If stopping, emit the event in any case to ensure cleanup.
-	if host == "" && flag != "stop" {
+	// Do not emit events without host (container is still being configured)
+	if host == "" {
 		return
 	}
 
-	// Collect all runtimes from status information.
+	// Collect all container IDs and runtimes from status information.
 	containerIDs := map[string]string{}
 	runtimes := map[string]string{}
 	for _, c := range containerstatuses {
@@ -165,18 +160,13 @@ func (p *Provider) emitEvents(pod *kubernetes.Pod, flag string, containers []*ku
 
 	// Emit container and port information
 	for _, c := range containers {
-		// If it doesn't have an ID, container doesn't exist in
-		// the runtime, emit only an event if we are stopping, so
-		// we are sure of cleaning up configurations.
 		cid := containerIDs[c.GetName()]
-		if cid == "" && flag != "stop" {
+
+		// If there is a container ID that is empty then ignore it. It either means that the container is still starting
+		// up or the container is shutting down.
+		if cid == "" {
 			continue
 		}
-
-		// This must be an id that doesn't depend on the state of the container
-		// so it works also on `stop` if containers have been already deleted.
-		eventID := fmt.Sprintf("%s.%s", pod.Metadata.GetUid(), c.GetName())
-
 		cmeta := common.MapStr{
 			"id":      cid,
 			"name":    c.GetName(),
@@ -200,7 +190,7 @@ func (p *Provider) emitEvents(pod *kubernetes.Pod, flag string, containers []*ku
 		if len(c.Ports) == 0 {
 			event := bus.Event{
 				"provider":   p.uuid,
-				"id":         eventID,
+				"id":         cid,
 				flag:         true,
 				"host":       host,
 				"kubernetes": kubemeta,
@@ -214,7 +204,7 @@ func (p *Provider) emitEvents(pod *kubernetes.Pod, flag string, containers []*ku
 		for _, port := range c.Ports {
 			event := bus.Event{
 				"provider":   p.uuid,
-				"id":         eventID,
+				"id":         cid,
 				flag:         true,
 				"host":       host,
 				"port":       port.GetContainerPort(),
@@ -275,12 +265,13 @@ func (p *Provider) generateHints(event bus.Event) bus.Event {
 	}
 
 	cname := builder.GetContainerName(container)
-	hints := builder.GenerateHints(annotations, cname, p.config.Prefix)
+	hints := builder.GenerateHints(annotations, cname, p.config.Prefix, p.config.DefaultDisable)
+	logp.Debug("kubernetes", "Generated hints %+v", hints)
 	if len(hints) != 0 {
 		e["hints"] = hints
 	}
 
-	logp.Debug("kubernetes", "Generated builder event %v", event)
+	logp.Debug("kubernetes", "Generated builder event %+v", e)
 
 	return e
 }