From ac1e16c327ae5dfb44a9d08151f1d397081cf926 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 28 Dec 2023 13:57:17 +0100 Subject: [PATCH] [8.12](backport #37458) Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37512) * Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration (#37458) * use InitDefaults(); elastic-agent-autodiscover: fix default resource metadata config - https://github.com/elastic/elastic-agent-autodiscover/pull/72 Signed-off-by: Tetiana Kravchenko * revert unintended changes Signed-off-by: Tetiana Kravchenko * upgrade elastic-agent-autodiscover lib to v0.6.6 Signed-off-by: Tetiana Kravchenko * run make update; fix linter issues Signed-off-by: Tetiana Kravchenko * revert changes related to the 'commonMetaConfig'; add missing metadata configuration on the metricset level Signed-off-by: Tetiana Kravchenko * x-pack/metricbeat: run make update Signed-off-by: Tetiana Kravchenko --------- Signed-off-by: Tetiana Kravchenko (cherry picked from commit 8c4a40fdbfda1a8e5ff2ceae01930a9f3e629d97) # Conflicts: # go.sum * fix merge conflicts Signed-off-by: Tetiana Kravchenko * Revert "fix merge conflicts" This reverts commit 88c08fc8700b953bb4b869aed171ccb462c0fd30. * fix conflicts only in the go.sum Signed-off-by: Tetiana Kravchenko --------- Signed-off-by: Tetiana Kravchenko Co-authored-by: Tetiana Kravchenko --- NOTICE.txt | 4 ++-- go.mod | 3 +-- go.sum | 4 ++-- .../add_kubernetes_metadata/config.go | 18 +++++++-------- .../add_kubernetes_metadata/config_test.go | 10 ++++---- .../add_kubernetes_metadata/kubernetes.go | 3 +-- metricbeat/docs/modules/kubernetes.asciidoc | 23 +++++++++++++++---- metricbeat/metricbeat.reference.yml | 23 +++++++++++++++---- .../kubernetes/_meta/config.reference.yml | 23 +++++++++++++++---- metricbeat/module/kubernetes/_meta/config.yml | 12 +++++++--- .../module/kubernetes/util/kubernetes.go | 20 ++++++++-------- metricbeat/modules.d/kubernetes.yml.disabled | 12 +++++++--- x-pack/metricbeat/metricbeat.reference.yml | 23 +++++++++++++++---- 13 files changed, 124 insertions(+), 54 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 85c993c9da1f..8ab54d1b2c7a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12268,11 +12268,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.5 +Version: v0.6.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 0daba82edbce..9b2dd55ae1a1 100644 --- a/go.mod +++ b/go.mod @@ -201,7 +201,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.5 + github.com/elastic/elastic-agent-autodiscover v0.6.6 github.com/elastic/elastic-agent-libs v0.7.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.9.1 @@ -413,7 +413,6 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 - ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index 20c032173063..28939e92c4b3 100644 --- a/go.sum +++ b/go.sum @@ -652,8 +652,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U= -github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= +github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA= github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0= github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 0998a275ea4f..7c74c82268d1 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -50,15 +50,13 @@ type Enabled struct { type PluginConfig []map[string]config.C -func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig { - return kubeAnnotatorConfig{ - SyncPeriod: 10 * time.Minute, - CleanupTimeout: 60 * time.Second, - DefaultMatchers: Enabled{true}, - DefaultIndexers: Enabled{true}, - Scope: "node", - AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), - } +func (k *kubeAnnotatorConfig) InitDefaults() { + k.SyncPeriod = 10 * time.Minute + k.CleanupTimeout = 60 * time.Second + k.DefaultMatchers = Enabled{true} + k.DefaultIndexers = Enabled{true} + k.Scope = "node" + k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig() } func (k *kubeAnnotatorConfig) Validate() error { @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error { err := matcherCfg.Unpack(&logsPathMatcher) if err != nil { - return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err) + return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err) } if logsPathMatcher.LogsPath == "" { return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well") diff --git a/libbeat/processors/add_kubernetes_metadata/config_test.go b/libbeat/processors/add_kubernetes_metadata/config_test.go index e94089f388a9..3857eb148fa6 100644 --- a/libbeat/processors/add_kubernetes_metadata/config_test.go +++ b/libbeat/processors/add_kubernetes_metadata/config_test.go @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) { for _, test := range tests { cfg := config.MustNewConfigFrom(test.cfg) - c := defaultKubernetesAnnotatorConfig() + var c kubeAnnotatorConfig err := cfg.Unpack(&c) if test.error { @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) { for _, test := range tests { cfg, _ := config.NewConfigFrom(test.matcherConfig) - c := defaultKubernetesAnnotatorConfig() - c.DefaultMatchers = Enabled{false} + var c kubeAnnotatorConfig - err := cfg.Unpack(&c) + _ = cfg.Unpack(&c) + c.DefaultMatchers = Enabled{false} c.Matchers = PluginConfig{ { test.matcherName: *cfg, }, } - err = c.Validate() + err := c.Validate() if test.error { require.NotNil(t, err) } else { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index a8667aef0a8a..954a59ab3f12 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) { - config := defaultKubernetesAnnotatorConfig() - + var config kubeAnnotatorConfig err := cfg.Unpack(&config) if err != nil { return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err) diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 6b0bbc023f3c..9ff079faa3be 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -232,11 +232,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -276,13 +283,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3588aaec9769..d6b8b9e9475d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -504,11 +504,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -548,13 +555,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index dcd59309119d..23f5ce8dea62 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -18,11 +18,18 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -62,13 +69,21 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 44ef19c97862..1c56e57b167f 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -16,15 +16,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 26728fccdaea..60b3360ab891 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // GetPodMetaGen requires cfg of type Config + // commonMetaConfig stores the metadata configuration of the resource itself commonMetaConfig := metadata.Config{} if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { logp.Err("Error initializing Kubernetes metadata enricher: %s", err) @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher( // update func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) - id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all + id := join(accessor.GetNamespace(), accessor.GetName()) switch r := r.(type) { case *kubernetes.Pod: @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher( return &nilEnricher{} } + // commonMetaConfig stores the metadata configuration of the resource itself + commonMetaConfig := metadata.Config{} + if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { + logp.Err("Error initializing Kubernetes metadata enricher: %s", err) + return &nilEnricher{} + } + cfg, _ := conf.NewConfigFrom(&commonMetaConfig) + // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher( } } - commonMetaConfig := metadata.Config{} - if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { - logp.Err("Error initializing Kubernetes metadata enricher: %s", err) - return &nilEnricher{} - } - cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata) enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { SyncPeriod: time.Minute * 10, AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), } + if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.New("error unpacking configs") } diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 23bd210a8357..12bbeee26ca5 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -19,15 +19,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 46e951ccb56e..a22db4f7f8cf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -901,11 +901,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -945,13 +952,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: