[8.12](backport #37458) Upgrade elastic-agent-autodiscover dependency…

…; use InitDefaults() to init default configuration (#37512)

* Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458)

* use InitDefaults(); elastic-agent-autodiscover: fix default resource metadata config - elastic/elastic-agent-autodiscover#72

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* revert unintended changes

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* upgrade elastic-agent-autodiscover lib to v0.6.6

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* run make update; fix linter issues

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* revert changes related to the 'commonMetaConfig'; add missing metadata configuration on the metricset level

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* x-pack/metricbeat: run make update

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

---------

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
(cherry picked from commit 8c4a40f)

# Conflicts:
#	go.sum

* fix merge conflicts

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* Revert "fix merge conflicts"

This reverts commit 88c08fc.

* fix conflicts only in the go.sum

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

---------

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
Co-authored-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

NOTICE.txt

@@ -12268,11 +12268,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-autodiscover
-Version: v0.6.5
+Version: v0.6.6
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

go.mod

@@ -201,7 +201,7 @@ require (
 	github.com/aws/smithy-go v1.13.5
 	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
 	github.com/elastic/bayeux v1.0.5
-	github.com/elastic/elastic-agent-autodiscover v0.6.5
+	github.com/elastic/elastic-agent-autodiscover v0.6.6
 	github.com/elastic/elastic-agent-libs v0.7.2
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.9.1
@@ -413,7 +413,6 @@ replace (
 	github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19
 	github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
 	k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15
-
 )
 
 // Exclude this version because the version has an invalid checksum.

go.sum

@@ -652,8 +652,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0=
 github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
 github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
-github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U=
-github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
+github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A=
+github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
 github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA=
 github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0=
 github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o=

libbeat/processors/add_kubernetes_metadata/config.go

@@ -50,15 +50,13 @@ type Enabled struct {
 
 type PluginConfig []map[string]config.C
 
-func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig {
-	return kubeAnnotatorConfig{
-		SyncPeriod:          10 * time.Minute,
-		CleanupTimeout:      60 * time.Second,
-		DefaultMatchers:     Enabled{true},
-		DefaultIndexers:     Enabled{true},
-		Scope:               "node",
-		AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
-	}
+func (k *kubeAnnotatorConfig) InitDefaults() {
+	k.SyncPeriod = 10 * time.Minute
+	k.CleanupTimeout = 60 * time.Second
+	k.DefaultMatchers = Enabled{true}
+	k.DefaultIndexers = Enabled{true}
+	k.Scope = "node"
+	k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig()
 }
 
 func (k *kubeAnnotatorConfig) Validate() error {
@@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error {
 
 				err := matcherCfg.Unpack(&logsPathMatcher)
 				if err != nil {
-					return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err)
+					return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err)
 				}
 				if logsPathMatcher.LogsPath == "" {
 					return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well")

libbeat/processors/add_kubernetes_metadata/config_test.go

@@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) {
 
 	for _, test := range tests {
 		cfg := config.MustNewConfigFrom(test.cfg)
-		c := defaultKubernetesAnnotatorConfig()
+		var c kubeAnnotatorConfig
 
 		err := cfg.Unpack(&c)
 		if test.error {
@@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) {
 	for _, test := range tests {
 		cfg, _ := config.NewConfigFrom(test.matcherConfig)
 
-		c := defaultKubernetesAnnotatorConfig()
-		c.DefaultMatchers = Enabled{false}
+		var c kubeAnnotatorConfig
 
-		err := cfg.Unpack(&c)
+		_ = cfg.Unpack(&c)
+		c.DefaultMatchers = Enabled{false}
 		c.Matchers = PluginConfig{
 			{
 				test.matcherName: *cfg,
 			},
 		}
-		err = c.Validate()
+		err := c.Validate()
 		if test.error {
 			require.NotNil(t, err)
 		} else {

libbeat/processors/add_kubernetes_metadata/kubernetes.go

@@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) {
 }
 
 func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) {
-	config := defaultKubernetesAnnotatorConfig()
-
+	var config kubeAnnotatorConfig
 	err := cfg.Unpack(&config)
 	if err != nil {
 		return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err)

metricbeat/docs/modules/kubernetes.asciidoc

@@ -232,11 +232,18 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
@@ -276,13 +283,21 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:

metricbeat/metricbeat.reference.yml

@@ -504,11 +504,18 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
@@ -548,13 +555,21 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:

metricbeat/module/kubernetes/_meta/config.reference.yml

@@ -18,11 +18,18 @@
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
@@ -62,13 +69,21 @@
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:

metricbeat/module/kubernetes/_meta/config.yml

@@ -16,15 +16,21 @@
 
   # Enriching parameters:
   #add_metadata: true
+  # If kube_config is not set, KUBECONFIG environment variable will be checked
+  # and if not present it will fall back to InCluster
+  #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
   #labels.dedot: true
   #annotations.dedot: true
+
   # When used outside the cluster:
   #node: node_name
-  # If kube_config is not set, KUBECONFIG environment variable will be checked
-  # and if not present it will fall back to InCluster
-  #kube_config: ~/.kube/config
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:

metricbeat/module/kubernetes/util/kubernetes.go

@@ -161,7 +161,7 @@ func NewResourceMetadataEnricher(
 		return &nilEnricher{}
 	}
 
-	// GetPodMetaGen requires cfg of type Config
+	// commonMetaConfig stores the metadata configuration of the resource itself
 	commonMetaConfig := metadata.Config{}
 	if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
 		logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
@@ -206,7 +206,7 @@ func NewResourceMetadataEnricher(
 		// update
 		func(m map[string]mapstr.M, r kubernetes.Resource) {
 			accessor, _ := meta.Accessor(r)
-			id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all
+			id := join(accessor.GetNamespace(), accessor.GetName())
 
 			switch r := r.(type) {
 			case *kubernetes.Pod:
@@ -308,6 +308,14 @@ func NewContainerMetadataEnricher(
 		return &nilEnricher{}
 	}
 
+	// commonMetaConfig stores the metadata configuration of the resource itself
+	commonMetaConfig := metadata.Config{}
+	if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
+		logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
+		return &nilEnricher{}
+	}
+	cfg, _ := conf.NewConfigFrom(&commonMetaConfig)
+
 	// Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to
 	// in order to be able to retrieve 2nd layer Owner metadata like in case of:
 	// Deployment -> Replicaset -> Pod
@@ -331,13 +339,6 @@ func NewContainerMetadataEnricher(
 		}
 	}
 
-	commonMetaConfig := metadata.Config{}
-	if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
-		logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
-		return &nilEnricher{}
-	}
-	cfg, _ := conf.NewConfigFrom(&commonMetaConfig)
-
 	metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata)
 
 	enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher,
@@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) {
 		SyncPeriod:          time.Minute * 10,
 		AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
 	}
+
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, errors.New("error unpacking configs")
 	}

metricbeat/modules.d/kubernetes.yml.disabled

@@ -19,15 +19,21 @@
 
   # Enriching parameters:
   #add_metadata: true
+  # If kube_config is not set, KUBECONFIG environment variable will be checked
+  # and if not present it will fall back to InCluster
+  #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
   #labels.dedot: true
   #annotations.dedot: true
+
   # When used outside the cluster:
   #node: node_name
-  # If kube_config is not set, KUBECONFIG environment variable will be checked
-  # and if not present it will fall back to InCluster
-  #kube_config: ~/.kube/config
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:

x-pack/metricbeat/metricbeat.reference.yml

@@ -901,11 +901,18 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata:
@@ -945,13 +952,21 @@ metricbeat.modules:
 
   # Enriching parameters:
   add_metadata: true
-  # When used outside the cluster:
-  #node: node_name
   # If kube_config is not set, KUBECONFIG environment variable will be checked
   # and if not present it will fall back to InCluster
   #kube_config: ~/.kube/config
+  #include_labels: []
+  #exclude_labels: []
+  #include_annotations: []
+  #labels.dedot: true
+  #annotations.dedot: true
+
+  # When used outside the cluster:
+  #node: node_name
+
   # Set the namespace to watch for resources
   #namespace: staging
+
   # To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
   # By default all labels will be included while annotations are not added by default.
   # add_resource_metadata: