From fb1a80d342730d1e2404525ed0b2ddee10c7673e Mon Sep 17 00:00:00 2001 From: Lee Hinman <57081003+leehinman@users.noreply.github.com> Date: Mon, 18 May 2020 15:41:34 -0500 Subject: [PATCH] [Filebeat] Improve ECS categorization field mappings in coredns module. (#18424) (#18586) * Improve ECS categorization field mappings in coredns module. - event.kind - event.category - event.type - event.outcome - network.protocol - related.ip - explicitly set ECS version Closes #16159 (cherry picked from commit 1ece3b68f78c66ef0c27d79a28b2b263d2a6b560) --- CHANGELOG.next.asciidoc | 1 + .../module/coredns/log/config/coredns.yml | 4 +++ .../coredns/log/ingest/pipeline-entry.yml | 28 +++++++++++++++ .../log/test/coredns-json.log-expected.json | 36 +++++++++++++++++++ .../log/test/coredns.log-expected.json | 24 +++++++++++++ 5 files changed, 93 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 36f62c5c2045..bc82f8b7efd4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -410,6 +410,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for v10, v11 and v12 logs on Postgres {issue}13810[13810] {pull}17732[17732] - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379] - The `logstash` module can now automatically detect the log file format (JSON or plaintext) and process it accordingly. {issue}9964[9964] {pull}18095[18095] +- Improve ECS categorization field mappings in coredns module. {issue}16159[16159] {pull}18424[18424] *Heartbeat* diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml index c085c9e3aab9..b2f0ebe45191 100644 --- a/x-pack/filebeat/module/coredns/log/config/coredns.yml +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -5,3 +5,7 @@ paths: {{ end }} tags: {{.tags}} processors: + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index 8f55838b21ea..44733495399c 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -107,6 +107,34 @@ processors: if (q.charAt(end) == (char) '.') { ctx.dns.question.name = q.substring(0, end); } + - set: + field: event.kind + value: event + - append: + field: event.category + value: network + - append: + field: event.type + value: protocol + - set: + field: event.outcome + value: success + if: "ctx?.dns?.response_code == 'NOERROR'" + - set: + field: event.outcome + value: failure + if: "ctx?.dns?.response_code != null && ctx.dns.response_code != 'NOERROR'" + - set: + field: network.protocol + value: dns + - append: + field: related.ip + value: "{{source.ip}}" + if: "ctx?.source?.ip != null" + - append: + field: related.ip + value: "{{destination.ip}}" + if: "ctx?.destination?.ip != null" on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json index 637f8cb0b4d2..23515a993cea 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -24,10 +24,18 @@ "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 102078, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \\\"A IN httpbin.org.cluster.local. udp 43 false 512\\\" NXDOMAIN qr,rd,ra 136 0.000102078s\", \"stream\": \"stdout\", \"time\": \"2019-02-12T00:27:28.903433597Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "failure", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -41,7 +49,11 @@ "log.level": "INFO", "log.offset": 0, "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.4" + ], "service.type": "coredns", "source.address": "172.17.0.4", "source.ip": "172.17.0.4", @@ -75,10 +87,18 @@ "dns.question.name": "httpbin.org", "dns.question.type": "A", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 82083, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \\\"A IN httpbin.org. udp 29 false 512\\\" NOERROR qr,rd,ra 83 0.000082083s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-19T02:57:23.214583742Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -92,7 +112,11 @@ "log.level": "INFO", "log.offset": 550, "message": "2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \"A IN httpbin.org. udp 29 false 512\" NOERROR qr,rd,ra 83 0.000082083s\n", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.9" + ], "service.type": "coredns", "source.address": "172.17.0.9", "source.ip": "172.17.0.9", @@ -126,10 +150,18 @@ "dns.question.name": "czbaoyu.com", "dns.question.type": "AAAA", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 62860, + "event.kind": "event", "event.module": "coredns", "event.original": "{\"message\":\"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \\\"AAAA IN czbaoyu.com. udp 29 false 512\\\" NOERROR qr,rd,ra 100 0.00006286s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-11T07:16:34.013970788Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -143,7 +175,11 @@ "log.level": "INFO", "log.offset": 1081, "message": "2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \"AAAA IN czbaoyu.com. udp 29 false 512\" NOERROR qr,rd,ra 100 0.00006286s\n", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "::1" + ], "service.type": "coredns", "source.address": "::1", "source.ip": "::1", diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json index ba3191a9e170..2d573602c17b 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -24,15 +24,27 @@ "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 102078, + "event.kind": "event", "event.module": "coredns", + "event.outcome": "failure", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "log.level": "INFO", "log.offset": 0, "message": "2019-03-06T08:55:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "172.17.0.4" + ], "service.type": "coredns", "source.address": "172.17.0.4", "source.ip": "172.17.0.4", @@ -66,15 +78,27 @@ "dns.question.name": "www.yahoo.com", "dns.question.type": "A", "dns.response_code": "NOERROR", + "event.category": [ + "network" + ], "event.dataset": "coredns.log", "event.duration": 20948545, + "event.kind": "event", "event.module": "coredns", + "event.outcome": "success", + "event.type": [ + "protocol" + ], "fileset.name": "log", "input.type": "log", "log.level": "INFO", "log.offset": 143, "message": "2019-03-18T22:13:36.289-07:00 [INFO] [::1]:57413 - 14639 \"A IN www.yahoo.com. udp 42 false 4096\" NOERROR qr,rd,ra 188 0.020948545s", + "network.protocol": "dns", "network.transport": "udp", + "related.ip": [ + "::1" + ], "service.type": "coredns", "source.address": "::1", "source.ip": "::1",