Enable keystore for autodiscover static configuration (elastic#16306)

(cherry picked from commit c1160e3)

CHANGELOG.next.asciidoc

@@ -263,6 +263,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `replace` processor for replacing string values of fields. {pull}17342[17342]
 - Add `urldecode` processor to for decoding URL-encoded fields. {pull}17505[17505]
 - Add support for AWS IAM `role_arn` in credentials config. {pull}17658[17658] {issue}12464[12464]
+- Add keystore support for autodiscover static configurations. {pull]16306[16306]
+- Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 
 *Auditbeat*
 

filebeat/autodiscover/builder/hints/logs.go

@@ -109,7 +109,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 		}
 		logp.Debug("hints.builder", "generated config %+v", configs)
 		// Apply information in event to the template to generate the final config
-		return template.ApplyConfigTemplate(event, configs)
+		return template.ApplyConfigTemplate(event, configs, false)
 	}
 
 	tempCfg := common.MapStr{}
@@ -163,7 +163,7 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 	logp.Debug("hints.builder", "generated config %+v", config)
 
 	// Apply information in event to the template to generate the final config
-	return template.ApplyConfigTemplate(event, []*common.Config{config})
+	return template.ApplyConfigTemplate(event, []*common.Config{config}, false)
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {

filebeat/beater/filebeat.go

@@ -410,6 +410,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 			),
 			autodiscover.QueryConfig(),
 			config.Autodiscover,
+			b.Keystore,
 		)
 		if err != nil {
 			return err

heartbeat/autodiscover/builder/hints/monitors.go

@@ -91,7 +91,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config {
 		}
 		hb.logger.Debugf("generated config %+v", configs)
 		// Apply information in event to the template to generate the final config
-		return template.ApplyConfigTemplate(event, configs)
+		return template.ApplyConfigTemplate(event, configs, false)
 	}
 
 	tempCfg := common.MapStr{}
@@ -121,7 +121,7 @@ func (hb *heartbeatHints) CreateConfig(event bus.Event) []*common.Config {
 	}
 
 	// Apply information in event to the template to generate the final config
-	return template.ApplyConfigTemplate(event, configs)
+	return template.ApplyConfigTemplate(event, configs, false)
 }
 
 func (hb *heartbeatHints) getType(hints common.MapStr) common.MapStr {

heartbeat/beater/heartbeat.go

@@ -156,7 +156,18 @@ func (bt *Heartbeat) RunReloadableMonitors(b *beat.Beat) (err error) {
 
 // makeAutodiscover creates an autodiscover object ready to be started.
 func (bt *Heartbeat) makeAutodiscover(b *beat.Beat) (*autodiscover.Autodiscover, error) {
-	return autodiscover.NewAutodiscover("heartbeat", b.Publisher, bt.dynamicFactory, autodiscover.QueryConfig(), bt.config.Autodiscover)
+	autodiscover, err := autodiscover.NewAutodiscover(
+		"heartbeat",
+		b.Publisher,
+		bt.dynamicFactory,
+		autodiscover.QueryConfig(),
+		bt.config.Autodiscover,
+		b.Keystore,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return autodiscover, nil
 }
 
 // Stop stops the beat.

libbeat/autodiscover/appenders/config/config.go

@@ -104,7 +104,7 @@ func (c *configAppender) Append(event bus.Event) {
 		}
 
 		// Apply the template
-		template.ApplyConfigTemplate(event, cfgs)
+		template.ApplyConfigTemplate(event, cfgs, false)
 	}
 
 	// Replace old config with newly appended configs

libbeat/autodiscover/autodiscover.go

@@ -29,6 +29,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/common/reload"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -72,6 +73,7 @@ func NewAutodiscover(
 	factory cfgfile.RunnerFactory,
 	configurer EventConfigurer,
 	config *Config,
+	keystore keystore.Keystore,
 ) (*Autodiscover, error) {
 	logger := logp.NewLogger("autodiscover")
 
@@ -81,7 +83,7 @@ func NewAutodiscover(
 	// Init providers
 	var providers []Provider
 	for _, providerCfg := range config.Providers {
-		provider, err := Registry.BuildProvider(bus, providerCfg)
+		provider, err := Registry.BuildProvider(bus, providerCfg, keystore)
 		if err != nil {
 			return nil, errors.Wrap(err, "error in autodiscover provider settings")
 		}
@@ -191,18 +193,15 @@ func (a *Autodiscover) handleStart(event bus.Event) bool {
 	if a.logger.IsDebug() {
 
 		for _, c := range configs {
-			rc := map[string]interface{}{}
-			c.Unpack(&rc)
-
-			a.logger.Debugf("Generated config: %+v", rc)
+			a.logger.Debugf("Generated config: %+v", common.DebugString(c, true))
 		}
 	}
 
 	meta := a.getMeta(event)
 	for _, config := range configs {
 		hash, err := cfgfile.HashConfig(config)
 		if err != nil {
-			a.logger.Debugf("Could not hash config %v: %v", config, err)
+			a.logger.Debugf("Could not hash config %v: %v", common.DebugString(config, true), err)
 			continue
 		}
 
@@ -216,7 +215,7 @@ func (a *Autodiscover) handleStart(event bus.Event) bool {
 		dynFields := a.meta.Store(hash, meta)
 
 		if a.configs[eventID][hash] != nil {
-			a.logger.Debugf("Config %v is already running", config)
+			a.logger.Debugf("Config %v is already running", common.DebugString(config, true))
 			continue
 		}
 

libbeat/autodiscover/autodiscover_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 	"github.com/elastic/beats/v7/libbeat/tests/resources"
 )
 
@@ -142,7 +143,7 @@ func TestAutodiscover(t *testing.T) {
 	// Register mock autodiscover provider
 	busChan := make(chan bus.Bus, 1)
 	Registry = NewRegistry()
-	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config) (Provider, error) {
+	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config, k keystore.Keystore) (Provider, error) {
 		// intercept bus to mock events
 		busChan <- b
 
@@ -164,9 +165,9 @@ func TestAutodiscover(t *testing.T) {
 	config := Config{
 		Providers: []*common.Config{providerConfig},
 	}
-
+	k, _ := keystore.NewFileKeystore("test")
 	// Create autodiscover manager
-	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config)
+	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config, k)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -266,7 +267,7 @@ func TestAutodiscoverHash(t *testing.T) {
 	busChan := make(chan bus.Bus, 1)
 
 	Registry = NewRegistry()
-	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config) (Provider, error) {
+	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config, k keystore.Keystore) (Provider, error) {
 		// intercept bus to mock events
 		busChan <- b
 
@@ -291,9 +292,9 @@ func TestAutodiscoverHash(t *testing.T) {
 	config := Config{
 		Providers: []*common.Config{providerConfig},
 	}
-
+	k, _ := keystore.NewFileKeystore("test")
 	// Create autodiscover manager
-	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config)
+	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config, k)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -332,7 +333,7 @@ func TestAutodiscoverWithConfigCheckFailures(t *testing.T) {
 	// Register mock autodiscover provider
 	busChan := make(chan bus.Bus, 1)
 	Registry = NewRegistry()
-	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config) (Provider, error) {
+	Registry.AddProvider("mock", func(b bus.Bus, uuid uuid.UUID, c *common.Config, k keystore.Keystore) (Provider, error) {
 		// intercept bus to mock events
 		busChan <- b
 
@@ -357,9 +358,9 @@ func TestAutodiscoverWithConfigCheckFailures(t *testing.T) {
 	config := Config{
 		Providers: []*common.Config{providerConfig},
 	}
-
+	k, _ := keystore.NewFileKeystore("test")
 	// Create autodiscover manager
-	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config)
+	autodiscover, err := NewAutodiscover("test", nil, &adapter, &adapter, &config, k)
 	if err != nil {
 		t.Fatal(err)
 	}

libbeat/autodiscover/provider.go

@@ -26,6 +26,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 )
 
 // Provider for autodiscover
@@ -34,7 +35,7 @@ type Provider interface {
 }
 
 // ProviderBuilder creates a new provider based on the given config and returns it
-type ProviderBuilder func(bus.Bus, uuid.UUID, *common.Config) (Provider, error)
+type ProviderBuilder func(bus.Bus, uuid.UUID, *common.Config, keystore.Keystore) (Provider, error)
 
 // AddProvider registers a new ProviderBuilder
 func (r *registry) AddProvider(name string, provider ProviderBuilder) error {
@@ -69,7 +70,7 @@ func (r *registry) GetProvider(name string) ProviderBuilder {
 }
 
 // BuildProvider reads provider configuration and instantiate one
-func (r *registry) BuildProvider(bus bus.Bus, c *common.Config) (Provider, error) {
+func (r *registry) BuildProvider(bus bus.Bus, c *common.Config, keystore keystore.Keystore) (Provider, error) {
 	var config ProviderConfig
 	err := c.Unpack(&config)
 	if err != nil {
@@ -86,5 +87,5 @@ func (r *registry) BuildProvider(bus bus.Bus, c *common.Config) (Provider, error
 		return nil, err
 	}
 
-	return builder(bus, uuid, c)
+	return builder(bus, uuid, c, keystore)
 }

libbeat/autodiscover/providers/docker/docker.go

@@ -33,6 +33,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/common/docker"
 	"github.com/elastic/beats/v7/libbeat/common/safemapstr"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -55,10 +56,11 @@ type Provider struct {
 	stoppers      map[string]*time.Timer
 	stopTrigger   chan *dockerContainerMetadata
 	logger        *logp.Logger
+	keystore      keystore.Keystore
 }
 
 // AutodiscoverBuilder builds and returns an autodiscover provider
-func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodiscover.Provider, error) {
+func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore keystore.Keystore) (autodiscover.Provider, error) {
 	logger := logp.NewLogger("docker")
 
 	errWrap := func(err error) error {
@@ -115,6 +117,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 		stoppers:      make(map[string]*time.Timer),
 		stopTrigger:   make(chan *dockerContainerMetadata),
 		logger:        logger,
+		keystore:      keystore,
 	}, nil
 }
 
@@ -303,6 +306,8 @@ func (d *Provider) emitContainer(container *docker.Container, meta *dockerMetada
 }
 
 func (d *Provider) publish(event bus.Event) {
+	// attach keystore to the event to be consumed by the static configs
+	event["keystore"] = d.keystore
 	// Try to match a config
 	if config := d.templates.GetConfig(event); config != nil {
 		event["config"] = config

libbeat/autodiscover/providers/docker/docker_integration_test.go

@@ -23,14 +23,14 @@ import (
 	"testing"
 	"time"
 
-	"github.com/elastic/beats/v7/libbeat/autodiscover/template"
-	"github.com/elastic/beats/v7/libbeat/logp"
-
 	"github.com/gofrs/uuid"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/beats/v7/libbeat/autodiscover/template"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
+	"github.com/elastic/beats/v7/libbeat/keystore"
+	"github.com/elastic/beats/v7/libbeat/logp"
 	dk "github.com/elastic/beats/v7/libbeat/tests/docker"
 )
 
@@ -53,7 +53,8 @@ func TestDockerStart(t *testing.T) {
 
 	s := &template.MapperSettings{nil, nil}
 	config.Templates = *s
-	provider, err := AutodiscoverBuilder(bus, UUID, common.MustNewConfigFrom(config))
+	k, _ := keystore.NewFileKeystore("test")
+	provider, err := AutodiscoverBuilder(bus, UUID, common.MustNewConfigFrom(config), k)
 	if err != nil {
 		t.Fatal(err)
 	}

libbeat/autodiscover/providers/jolokia/jolokia.go

@@ -27,6 +27,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/autodiscover/template"
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 )
 
 func init() {
@@ -48,11 +49,12 @@ type Provider struct {
 	appenders autodiscover.Appenders
 	templates template.Mapper
 	discovery DiscoveryProber
+	keystore  keystore.Keystore
 }
 
 // AutodiscoverBuilder builds a Jolokia Discovery autodiscover provider, it fails if
 // there is some problem with the configuration
-func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodiscover.Provider, error) {
+func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore keystore.Keystore) (autodiscover.Provider, error) {
 	errWrap := func(err error) error {
 		return errors.Wrap(err, "error setting up jolokia autodiscover provider")
 	}
@@ -92,6 +94,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 		builders:  builders,
 		appenders: appenders,
 		discovery: discovery,
+		keystore:  keystore,
 	}, nil
 }
 
@@ -106,6 +109,8 @@ func (p *Provider) Start() {
 }
 
 func (p *Provider) publish(event bus.Event) {
+	// attach keystore to the event to be consumed by the static configs
+	event["keystore"] = p.keystore
 	if config := p.templates.GetConfig(event); config != nil {
 		event["config"] = config
 	} else if config := p.builders.GetConfig(event); config != nil {

libbeat/autodiscover/providers/kubernetes/kubernetes.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/common/bus"
 	"github.com/elastic/beats/v7/libbeat/common/kubernetes"
+	"github.com/elastic/beats/v7/libbeat/keystore"
 	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
@@ -54,10 +55,11 @@ type Provider struct {
 	appenders autodiscover.Appenders
 	logger    *logp.Logger
 	eventer   Eventer
+	keystore  keystore.Keystore
 }
 
 // AutodiscoverBuilder builds and returns an autodiscover provider
-func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodiscover.Provider, error) {
+func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config, keystore keystore.Keystore) (autodiscover.Provider, error) {
 	logger := logp.NewLogger("autodiscover")
 
 	errWrap := func(err error) error {
@@ -97,6 +99,7 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 		builders:  builders,
 		appenders: appenders,
 		logger:    logger,
+		keystore:  keystore,
 	}
 
 	switch config.Resource {
@@ -135,6 +138,8 @@ func (p *Provider) String() string {
 }
 
 func (p *Provider) publish(event bus.Event) {
+	// attach keystore to the event to be consumed by the static configs
+	event["keystore"] = p.keystore
 	// Try to match a config
 	if config := p.templates.GetConfig(event); config != nil {
 		event["config"] = config