From 3b35e38c9862cf0ea1e073f449693b07286c3642 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 2 Feb 2019 09:50:00 -0800 Subject: [PATCH 1/5] Address issue #10505 - add_kubernetes_metadata processor is broken because it uses old field source for lookup --- CHANGELOG.asciidoc | 2 ++ filebeat/processor/add_kubernetes_metadata/matchers.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 4cbb83a60fcd..adb12dd12473 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Add module zeek. {issue}9931[9931] {pull}10034[10034] - Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] +- Address add_kubernetes_metadata processor issue where old source field is + still used for matcher. {issue}10505[10505] *Heartbeat* diff --git a/filebeat/processor/add_kubernetes_metadata/matchers.go b/filebeat/processor/add_kubernetes_metadata/matchers.go index bac493543543..c984c0f60e6e 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers.go @@ -77,9 +77,9 @@ const containerIdLen = 64 const podUIDPos = 5 func (f *LogPathMatcher) MetadataIndex(event common.MapStr) string { - if value, ok := event["source"]; ok { + if value, ok := event["log"].(common.MapStr)["file"].(common.MapStr)["path"]; ok { source := value.(string) - logp.Debug("kubernetes", "Incoming source value: %s", source) + logp.Debug("kubernetes", "Incoming log.file.path value: %s", source) if !strings.Contains(source, f.LogsPath) { logp.Debug("kubernetes", "Error extracting container id - source value does not contain matcher's logs_path '%s'.", f.LogsPath) From dd762d6333c85e2dd82de8994b5dfe0e42b8d58d Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 2 Feb 2019 14:00:49 -0800 Subject: [PATCH 2/5] Address issue 10505 - update documentation --- libbeat/docs/processors-using.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index c8fe56fdbe9e..4332cd741325 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -755,7 +755,8 @@ construct a lookup key with the value of the field `metricset.host`. Each Beat can define its own default indexers and matchers which are enabled by default. For example, FileBeat enables the `container` indexer, which indexes pod metadata based on all container IDs, and a `logs_path` matcher, which takes -the `source` field, extracts the container ID, and uses it to retrieve metadata. +the `log.file.path` field, extracts the container ID, and uses it to retrieve +metadata. The configuration below enables the processor when {beatname_lc} is run as a pod in Kubernetes. From 58a2df0eaffb875f37915bfac8d0d375fcf68d53 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Sat, 2 Feb 2019 14:02:28 -0800 Subject: [PATCH 3/5] Add pull # to CHANGELOG.asciidoc --- CHANGELOG.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index adb12dd12473..3c0fa67dbe0a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -29,7 +29,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add module zeek. {issue}9931[9931] {pull}10034[10034] - Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] - Address add_kubernetes_metadata processor issue where old source field is - still used for matcher. {issue}10505[10505] + still used for matcher. {issue}10505[10505] {pull}10506[10506] *Heartbeat* From f27b50ec543ead7e4e6cfa536cef583442dc0b46 Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 4 Feb 2019 11:38:16 -0800 Subject: [PATCH 4/5] Fix unit test case --- filebeat/processor/add_kubernetes_metadata/matchers_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/filebeat/processor/add_kubernetes_metadata/matchers_test.go b/filebeat/processor/add_kubernetes_metadata/matchers_test.go index 2a8b0bf89d66..090094686509 100644 --- a/filebeat/processor/add_kubernetes_metadata/matchers_test.go +++ b/filebeat/processor/add_kubernetes_metadata/matchers_test.go @@ -108,7 +108,11 @@ func executeTestWithResourceType(t *testing.T, cfgLogsPath string, cfgResourceTy assert.Nil(t, err) input := common.MapStr{ - "source": source, + "log": common.MapStr{ + "file": common.MapStr{ + "path": source, + }, + }, } output := logMatcher.MetadataIndex(input) assert.Equal(t, expectedResult, output) From 88fadcca22ff31994a2b89861040c1487f41370f Mon Sep 17 00:00:00 2001 From: Ray Qiu Date: Mon, 4 Feb 2019 13:16:37 -0800 Subject: [PATCH 5/5] Move update to CHANGELOG.next.asciidoc --- CHANGELOG.asciidoc | 2 -- CHANGELOG.next.asciidoc | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 3c0fa67dbe0a..4cbb83a60fcd 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,8 +28,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005] - Add module zeek. {issue}9931[9931] {pull}10034[10034] - Rename many `kibana.log.*` fields to map to ECS. {pull}9301[9301] -- Address add_kubernetes_metadata processor issue where old source field is - still used for matcher. {issue}10505[10505] {pull}10506[10506] *Heartbeat* diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index df9a7f9116af..c5f1bfeb69aa 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -77,6 +77,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - The `elasticsearch/deprecation` fileset now indexes the `component` field under `elasticsearch` instead of `elasticsearch.server`. {pull}10445[10445] - Remove field `kafka.log.trace.full` from kafka.log fielset. {pull}10398[10398] - Change field `kafka.log.class` for kafka.log fileset from text to keyword. {pull}10398[10398] +- Address add_kubernetes_metadata processor issue where old source field is + still used for matcher. {issue}10505[10505] {pull}10506[10506] *Heartbeat*