From 634c81b8f24d2d6ea7f498e8e28d95e6085b45fb Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 12 Mar 2020 12:53:53 +0200 Subject: [PATCH 1/2] Fix k8s metadata issue (#16834) (cherry picked from commit 034ee554ef6849fafea21b72db228f3c7d0fc159) --- CHANGELOG.next.asciidoc | 2 + .../common/kubernetes/metadata/metadata.go | 10 +++++ .../common/kubernetes/metadata/namespace.go | 26 ++++++++----- .../kubernetes/metadata/namespace_test.go | 23 +++++------ .../common/kubernetes/metadata/node_test.go | 12 +++--- libbeat/common/kubernetes/metadata/pod.go | 22 +---------- .../common/kubernetes/metadata/resource.go | 4 +- .../kubernetes/metadata/resource_test.go | 12 +++--- .../kubernetes/metadata/service_test.go | 39 ++++++++++++------- 9 files changed, 80 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f8ce4e5dad93..375b3d19ac02 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -85,6 +85,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix `NewContainerMetadataEnricher` to use default config for kubernetes module. {pull}16857[16857] +- Improve some logging messages for add_kubernetes_metadata processor {pull}16866[16866] +- Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834] *Auditbeat* diff --git a/libbeat/common/kubernetes/metadata/metadata.go b/libbeat/common/kubernetes/metadata/metadata.go index d81f79f28d58..a2b474281cfd 100644 --- a/libbeat/common/kubernetes/metadata/metadata.go +++ b/libbeat/common/kubernetes/metadata/metadata.go @@ -18,6 +18,8 @@ package metadata import ( + "strings" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/kubernetes" "github.com/elastic/beats/v7/libbeat/common/safemapstr" @@ -40,3 +42,11 @@ func WithFields(key string, value interface{}) FieldOptions { safemapstr.Put(meta, key, value) } } + +// WithLabels FieldOption allows adding labels under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithLabels(kind string) FieldOptions { + return func(meta common.MapStr) { + safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } +} diff --git a/libbeat/common/kubernetes/metadata/namespace.go b/libbeat/common/kubernetes/metadata/namespace.go index 92dfbc55b2af..dd3011a6c471 100644 --- a/libbeat/common/kubernetes/metadata/namespace.go +++ b/libbeat/common/kubernetes/metadata/namespace.go @@ -60,17 +60,16 @@ func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) common.M return nil } - obj, ok, _ := n.store.GetByKey(name) - if !ok { - return nil - } + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } - no, ok := obj.(*kubernetes.Namespace) - if !ok { - return nil + return n.Generate(no, opts...) } - return n.Generate(no, opts...) + return nil } func flattenMetadata(in common.MapStr) common.MapStr { @@ -84,7 +83,6 @@ func flattenMetadata(in common.MapStr) common.MapStr { if !ok { return nil } - for k, v := range fields { if k == "name" { out[resource] = v @@ -93,5 +91,15 @@ func flattenMetadata(in common.MapStr) common.MapStr { } } + rawLabels, err := in.GetValue("labels") + if err != nil { + return out + } + labels, ok := rawLabels.(common.MapStr) + if !ok { + return out + } + out[resource+"_labels"] = labels + return out } diff --git a/libbeat/common/kubernetes/metadata/namespace_test.go b/libbeat/common/kubernetes/metadata/namespace_test.go index 5ac460d7ca77..40763887197d 100644 --- a/libbeat/common/kubernetes/metadata/namespace_test.go +++ b/libbeat/common/kubernetes/metadata/namespace_test.go @@ -58,17 +58,18 @@ func TestNamespace_Generate(t *testing.T) { }, }, // Use this for 8.0 - /*output: common.MapStr{ - "namespace": common.MapStr{ - "name": "obj", - "uid": uid, - "labels": common.MapStr{ - "foo": "bar", + /* + output: common.MapStr{ + "namespace": common.MapStr{ + "name": name, + "uid": uid, + "labels": common.MapStr{ + "foo": "bar", + }, }, - }, - },*/ + },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", @@ -114,7 +115,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { /* output: common.MapStr{ "namespace": common.MapStr{ - "name": "obj", + "name": name, "uid": uid, "labels": common.MapStr{ "foo": "bar", @@ -122,7 +123,7 @@ func TestNamespace_GenerateFromName(t *testing.T) { }, },*/ output: common.MapStr{ - "namespace": "obj", + "namespace": name, "namespace_uid": uid, "namespace_labels": common.MapStr{ "foo": "bar", diff --git a/libbeat/common/kubernetes/metadata/node_test.go b/libbeat/common/kubernetes/metadata/node_test.go index 226e84710c24..100db49c6d8a 100644 --- a/libbeat/common/kubernetes/metadata/node_test.go +++ b/libbeat/common/kubernetes/metadata/node_test.go @@ -61,9 +61,9 @@ func TestNode_Generate(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, @@ -106,9 +106,9 @@ func TestNode_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, }, }, diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index c25db1f7a76a..9234f54533b7 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,11 +49,9 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) - // TODO: remove this call when moving to 8.0 - out = p.exportPodLabelsAndAnnotations(out) if p.node != nil { - meta := p.node.GenerateFromName(po.Spec.NodeName) + meta := p.node.GenerateFromName(po.Spec.NodeName, WithLabels("node")) if meta != nil { out.Put("node", meta["node"]) } else { @@ -91,21 +89,3 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } - -func (p *pod) exportPodLabelsAndAnnotations(in common.MapStr) common.MapStr { - labels, err := in.GetValue("pod.labels") - if err != nil { - return in - } - in.Put("labels", labels) - in.Delete("pod.labels") - - annotations, err := in.GetValue("pod.annotations") - if err != nil { - return in - } - in.Put("annotations", annotations) - in.Delete("pod.annotations") - - return in -} diff --git a/libbeat/common/kubernetes/metadata/resource.go b/libbeat/common/kubernetes/metadata/resource.go index e4f4b8ca0289..f8f986fe42c4 100644 --- a/libbeat/common/kubernetes/metadata/resource.go +++ b/libbeat/common/kubernetes/metadata/resource.go @@ -98,11 +98,11 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, options ...Fie } if len(labelMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".labels", labelMap) + safemapstr.Put(meta, "labels", labelMap) } if len(annotationsMap) != 0 { - safemapstr.Put(meta, strings.ToLower(kind)+".annotations", annotationsMap) + safemapstr.Put(meta, "annotations", annotationsMap) } for _, option := range options { diff --git a/libbeat/common/kubernetes/metadata/resource_test.go b/libbeat/common/kubernetes/metadata/resource_test.go index 6df542742f07..f4c25ecbad5a 100644 --- a/libbeat/common/kubernetes/metadata/resource_test.go +++ b/libbeat/common/kubernetes/metadata/resource_test.go @@ -62,9 +62,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -99,9 +99,9 @@ func TestResource_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ diff --git a/libbeat/common/kubernetes/metadata/service_test.go b/libbeat/common/kubernetes/metadata/service_test.go index a574f1a7b7a1..0fc06af3a0be 100644 --- a/libbeat/common/kubernetes/metadata/service_test.go +++ b/libbeat/common/kubernetes/metadata/service_test.go @@ -64,9 +64,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -101,9 +101,9 @@ func TestService_Generate(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -153,9 +153,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", }, @@ -190,9 +190,9 @@ func TestService_GenerateFromName(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "deployment": common.MapStr{ @@ -262,10 +262,19 @@ func TestService_GenerateWithNamespace(t *testing.T) { "service": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, + "labels": common.MapStr{ + "foo": "bar", + }, + // Use this for 8.0 + /* + "namespace": common.MapStr{ + "name": "default", + "uid": uid, + "labels": common.MapStr{ + "nskey": "nsvalue", + }, + },*/ "namespace": "default", "namespace_uid": uid, "namespace_labels": common.MapStr{ From 21e0ede64e67d5bef973e3ea9da3ce9c91d80cde Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Thu, 12 Mar 2020 13:04:01 +0200 Subject: [PATCH 2/2] Update CHANGELOG.next.asciidoc --- CHANGELOG.next.asciidoc | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 375b3d19ac02..94ba6134545f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -85,7 +85,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix `NewContainerMetadataEnricher` to use default config for kubernetes module. {pull}16857[16857] -- Improve some logging messages for add_kubernetes_metadata processor {pull}16866[16866] - Fix k8s metadata issue regarding node labels not shown up on root level of metadata. {pull}16834[16834] *Auditbeat*