Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.12](backport #37458) Upgrade elastic-agent-autodiscover dependency; use InitDefaults() to init default configuration #37512

Merged
merged 5 commits into from
Dec 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions NOTICE.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12268,11 +12268,11 @@ SOFTWARE.

--------------------------------------------------------------------------------
Dependency : github.com/elastic/elastic-agent-autodiscover
Version: v0.6.5
Version: v0.6.6
Licence type (autodetected): Apache-2.0
--------------------------------------------------------------------------------

Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE:
Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE:

Apache License
Version 2.0, January 2004
Expand Down
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ require (
github.com/aws/smithy-go v1.13.5
github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5
github.com/elastic/bayeux v1.0.5
github.com/elastic/elastic-agent-autodiscover v0.6.5
github.com/elastic/elastic-agent-autodiscover v0.6.6
github.com/elastic/elastic-agent-libs v0.7.2
github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
github.com/elastic/elastic-agent-system-metrics v0.9.1
Expand Down Expand Up @@ -413,7 +413,6 @@ replace (
github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19
github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c
k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15

)

// Exclude this version because the version has an invalid checksum.
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -652,8 +652,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0=
github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs=
github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY=
github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U=
github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A=
github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o=
github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA=
github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0=
github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o=
Expand Down
18 changes: 8 additions & 10 deletions libbeat/processors/add_kubernetes_metadata/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,13 @@ type Enabled struct {

type PluginConfig []map[string]config.C

func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig {
return kubeAnnotatorConfig{
SyncPeriod: 10 * time.Minute,
CleanupTimeout: 60 * time.Second,
DefaultMatchers: Enabled{true},
DefaultIndexers: Enabled{true},
Scope: "node",
AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
}
func (k *kubeAnnotatorConfig) InitDefaults() {
k.SyncPeriod = 10 * time.Minute
k.CleanupTimeout = 60 * time.Second
k.DefaultMatchers = Enabled{true}
k.DefaultIndexers = Enabled{true}
k.Scope = "node"
k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig()
}

func (k *kubeAnnotatorConfig) Validate() error {
Expand All @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error {

err := matcherCfg.Unpack(&logsPathMatcher)
if err != nil {
return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err)
return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err)
}
if logsPathMatcher.LogsPath == "" {
return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well")
Expand Down
10 changes: 5 additions & 5 deletions libbeat/processors/add_kubernetes_metadata/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) {

for _, test := range tests {
cfg := config.MustNewConfigFrom(test.cfg)
c := defaultKubernetesAnnotatorConfig()
var c kubeAnnotatorConfig

err := cfg.Unpack(&c)
if test.error {
Expand Down Expand Up @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) {
for _, test := range tests {
cfg, _ := config.NewConfigFrom(test.matcherConfig)

c := defaultKubernetesAnnotatorConfig()
c.DefaultMatchers = Enabled{false}
var c kubeAnnotatorConfig

err := cfg.Unpack(&c)
_ = cfg.Unpack(&c)
c.DefaultMatchers = Enabled{false}
c.Matchers = PluginConfig{
{
test.matcherName: *cfg,
},
}
err = c.Validate()
err := c.Validate()
if test.error {
require.NotNil(t, err)
} else {
Expand Down
3 changes: 1 addition & 2 deletions libbeat/processors/add_kubernetes_metadata/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) {
}

func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) {
config := defaultKubernetesAnnotatorConfig()

var config kubeAnnotatorConfig
err := cfg.Unpack(&config)
if err != nil {
return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err)
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/docs/modules/kubernetes.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -232,11 +232,18 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -276,13 +283,21 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -504,11 +504,18 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -548,13 +555,21 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
23 changes: 19 additions & 4 deletions metricbeat/module/kubernetes/_meta/config.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,18 @@

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -62,13 +69,21 @@

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
12 changes: 9 additions & 3 deletions metricbeat/module/kubernetes/_meta/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,21 @@

# Enriching parameters:
#add_metadata: true
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
20 changes: 11 additions & 9 deletions metricbeat/module/kubernetes/util/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher(
return &nilEnricher{}
}

// GetPodMetaGen requires cfg of type Config
// commonMetaConfig stores the metadata configuration of the resource itself
commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
Expand Down Expand Up @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher(
// update
func(m map[string]mapstr.M, r kubernetes.Resource) {
accessor, _ := meta.Accessor(r)
id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all
id := join(accessor.GetNamespace(), accessor.GetName())

switch r := r.(type) {
case *kubernetes.Pod:
Expand Down Expand Up @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher(
return &nilEnricher{}
}

// commonMetaConfig stores the metadata configuration of the resource itself
commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
return &nilEnricher{}
}
cfg, _ := conf.NewConfigFrom(&commonMetaConfig)

// Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to
// in order to be able to retrieve 2nd layer Owner metadata like in case of:
// Deployment -> Replicaset -> Pod
Expand All @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher(
}
}

commonMetaConfig := metadata.Config{}
if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil {
logp.Err("Error initializing Kubernetes metadata enricher: %s", err)
return &nilEnricher{}
}
cfg, _ := conf.NewConfigFrom(&commonMetaConfig)

metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata)

enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher,
Expand Down Expand Up @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) {
SyncPeriod: time.Minute * 10,
AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(),
}

if err := base.Module().UnpackConfig(&config); err != nil {
return nil, errors.New("error unpacking configs")
}
Expand Down
12 changes: 9 additions & 3 deletions metricbeat/modules.d/kubernetes.yml.disabled
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,21 @@

# Enriching parameters:
#add_metadata: true
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
23 changes: 19 additions & 4 deletions x-pack/metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -901,11 +901,18 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down Expand Up @@ -945,13 +952,21 @@ metricbeat.modules:

# Enriching parameters:
add_metadata: true
# When used outside the cluster:
#node: node_name
# If kube_config is not set, KUBECONFIG environment variable will be checked
# and if not present it will fall back to InCluster
#kube_config: ~/.kube/config
#include_labels: []
#exclude_labels: []
#include_annotations: []
#labels.dedot: true
#annotations.dedot: true

# When used outside the cluster:
#node: node_name

# Set the namespace to watch for resources
#namespace: staging

# To configure additionally node and namespace metadata `add_resource_metadata` can be defined.
# By default all labels will be included while annotations are not added by default.
# add_resource_metadata:
Expand Down
Loading