From 4863c4470692f8ef58bfcef85b772afbfb5cc8cc Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 25 Feb 2020 12:13:06 +0200 Subject: [PATCH] Fix k8s pod labels tier in metadata (#16480) (cherry picked from commit 837279a4b5c2b5e560bfc8fc3e6dd7d4c1cebd12) --- CHANGELOG.next.asciidoc | 15 ++++++++++ libbeat/common/kubernetes/metadata/pod.go | 13 ++++++++ .../common/kubernetes/metadata/pod_test.go | 30 +++++++++---------- .../add_kubernetes_metadata/indexers_test.go | 30 +++++++++---------- 4 files changed, 58 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16760d5357dd..80e199f95aab 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -39,6 +39,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* +- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] +- Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568] +- Fix missing output in dockerlogbeat {pull}15719[15719] +- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442] +- Do not load dashboards where not available. {pull}15802[15802] +- Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516} +- Update replicaset group to apps/v1 {pull}15854[15802] +- Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708] +- Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937} +- Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] +- Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] +- Fix loading processors from annotation hints. {pull}16348[16348] +- Fix k8s pods labels broken schema. {pull}16480[16480] +- Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} *Auditbeat* diff --git a/libbeat/common/kubernetes/metadata/pod.go b/libbeat/common/kubernetes/metadata/pod.go index b42fa005cc05..7385d2abdf5b 100644 --- a/libbeat/common/kubernetes/metadata/pod.go +++ b/libbeat/common/kubernetes/metadata/pod.go @@ -49,6 +49,8 @@ func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) common.Map } out := p.resource.Generate("pod", obj, opts...) + // TODO: remove this call when moving to 8.0 + out = p.exportPodLabels(out) if p.node != nil { meta := p.node.GenerateFromName(po.Spec.NodeName) @@ -89,3 +91,14 @@ func (p *pod) GenerateFromName(name string, opts ...FieldOptions) common.MapStr return nil } + +func (p *pod) exportPodLabels(in common.MapStr) common.MapStr { + labels, err := in.GetValue("pod.labels") + if err != nil { + return in + } + in.Put("labels", labels) + in.Delete("pod.labels") + + return in +} diff --git a/libbeat/common/kubernetes/metadata/pod_test.go b/libbeat/common/kubernetes/metadata/pod_test.go index 2c42958a759f..c3838ad2c328 100644 --- a/libbeat/common/kubernetes/metadata/pod_test.go +++ b/libbeat/common/kubernetes/metadata/pod_test.go @@ -67,9 +67,9 @@ func TestPod_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, + }, + "labels": common.MapStr{ + "foo": "bar", }, "namespace": "default", "node": common.MapStr{ @@ -110,9 +110,6 @@ func TestPod_Generate(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "deployment": common.MapStr{ @@ -121,6 +118,9 @@ func TestPod_Generate(t *testing.T) { "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } @@ -168,14 +168,14 @@ func TestPod_GenerateFromName(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, { @@ -211,9 +211,6 @@ func TestPod_GenerateFromName(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "deployment": common.MapStr{ @@ -222,6 +219,9 @@ func TestPod_GenerateFromName(t *testing.T) { "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } @@ -304,9 +304,6 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { "pod": common.MapStr{ "name": "obj", "uid": uid, - "labels": common.MapStr{ - "foo": "bar", - }, }, "namespace": "default", "namespace_uid": uid, @@ -320,6 +317,9 @@ func TestPod_GenerateWithNodeNamespace(t *testing.T) { "nodekey": "nodevalue", }, }, + "labels": common.MapStr{ + "foo": "bar", + }, }, }, } diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index 0d74cece0122..acbe1bf51eb7 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -66,9 +66,9 @@ func TestPodIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, + }, + "labels": common.MapStr{ + "labelkey": "labelvalue", }, "namespace": "testns", "node": common.MapStr{ @@ -117,14 +117,14 @@ func TestPodUIDIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } assert.Equal(t, expected.String(), indexers[0].Data.String()) @@ -173,14 +173,14 @@ func TestContainerIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } container1 := "docker://abcde" pod.Spec.NodeName = nodeName @@ -250,7 +250,7 @@ func TestFilteredGenMeta(t *testing.T) { indexers := podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ := indexers[0].Data.GetValue("pod.labels") + rawLabels, _ := indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok := rawLabels.(common.MapStr) @@ -274,7 +274,7 @@ func TestFilteredGenMeta(t *testing.T) { indexers = podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ = indexers[0].Data.GetValue("pod.labels") + rawLabels, _ = indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok = rawLabels.(common.MapStr) @@ -331,7 +331,7 @@ func TestFilteredGenMetaExclusion(t *testing.T) { indexers := podIndexer.GetMetadata(&pod) assert.Equal(t, len(indexers), 1) - rawLabels, _ := indexers[0].Data.GetValue("pod.labels") + rawLabels, _ := indexers[0].Data.GetValue("labels") assert.NotNil(t, rawLabels) labelMap, ok := rawLabels.(common.MapStr) @@ -392,14 +392,14 @@ func TestIpPortIndexer(t *testing.T) { "pod": common.MapStr{ "name": "testpod", "uid": "005f3b90-4b9d-12f8-acf0-31020a840133", - "labels": common.MapStr{ - "labelkey": "labelvalue", - }, }, "namespace": "testns", "node": common.MapStr{ "name": "testnode", }, + "labels": common.MapStr{ + "labelkey": "labelvalue", + }, } pod.Spec.Containers = []v1.Container{