[Filebeat] Improve ECS categorization field mappings in coredns modul…

…e. (elastic#18424) * Improve ECS categorization field mappings in coredns module. - event.kind - event.category - event.type - event.outcome - network.protocol - related.ip - explicitly set ECS version Closes elastic#16159 (cherry picked from commit 1ece3b6)
leehinman · May 15, 2020 · 954dea7 · 954dea7
1 parent 1b3a5fb
commit 954dea7
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 0 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -407,6 +407,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for v10, v11 and v12 logs on Postgres {issue}13810[13810] {pull}17732[17732]
 - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379]
 - The `logstash` module can now automatically detect the log file format (JSON or plaintext) and process it accordingly. {issue}9964[9964] {pull}18095[18095]
+- Improve ECS categorization field mappings in coredns module. {issue}16159[16159] {pull}18424[18424]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml
@@ -5,3 +5,7 @@ paths:
 {{ end }}
 tags: {{.tags}}
 processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0
diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml
@@ -107,6 +107,34 @@ processors:
         if (q.charAt(end) == (char) '.') {
             ctx.dns.question.name = q.substring(0, end);
         }
+  - set:
+      field: event.kind
+      value: event
+  - append:
+      field: event.category
+      value: network
+  - append:
+      field: event.type
+      value: protocol
+  - set:
+      field: event.outcome
+      value: success
+      if: "ctx?.dns?.response_code == 'NOERROR'"
+  - set:
+      field: event.outcome
+      value: failure
+      if: "ctx?.dns?.response_code != null && ctx.dns.response_code != 'NOERROR'"
+  - set:
+      field: network.protocol
+      value: dns
+  - append:
+      field: related.ip
+      value: "{{source.ip}}"
+      if: "ctx?.source?.ip != null"
+  - append:
+      field: related.ip
+      value: "{{destination.ip}}"
+      if: "ctx?.destination?.ip != null"
 on_failure:
   - set:
       field: error.message

diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json
@@ -24,10 +24,18 @@
         "dns.question.name": "httpbin.org.cluster.local",
         "dns.question.type": "A",
         "dns.response_code": "NXDOMAIN",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "coredns.log",
         "event.duration": 102078,
+        "event.kind": "event",
         "event.module": "coredns",
         "event.original": "{\"message\":\"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \\\"A IN httpbin.org.cluster.local. udp 43 false 512\\\" NXDOMAIN qr,rd,ra 136 0.000102078s\", \"stream\": \"stdout\", \"time\": \"2019-02-12T00:27:28.903433597Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }",
+        "event.outcome": "failure",
+        "event.type": [
+            "protocol"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "kubernetes.container.name": "coredns",
@@ -41,7 +49,11 @@
         "log.level": "INFO",
         "log.offset": 0,
         "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s",
+        "network.protocol": "dns",
         "network.transport": "udp",
+        "related.ip": [
+            "172.17.0.4"
+        ],
         "service.type": "coredns",
         "source.address": "172.17.0.4",
         "source.ip": "172.17.0.4",
@@ -75,10 +87,18 @@
         "dns.question.name": "httpbin.org",
         "dns.question.type": "A",
         "dns.response_code": "NOERROR",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "coredns.log",
         "event.duration": 82083,
+        "event.kind": "event",
         "event.module": "coredns",
         "event.original": "{\"message\":\"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \\\"A IN httpbin.org. udp 29 false 512\\\" NOERROR qr,rd,ra 83 0.000082083s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-19T02:57:23.214583742Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }",
+        "event.outcome": "success",
+        "event.type": [
+            "protocol"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "kubernetes.container.name": "coredns",
@@ -92,7 +112,11 @@
         "log.level": "INFO",
         "log.offset": 550,
         "message": "2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \"A IN httpbin.org. udp 29 false 512\" NOERROR qr,rd,ra 83 0.000082083s\n",
+        "network.protocol": "dns",
         "network.transport": "udp",
+        "related.ip": [
+            "172.17.0.9"
+        ],
         "service.type": "coredns",
         "source.address": "172.17.0.9",
         "source.ip": "172.17.0.9",
@@ -126,10 +150,18 @@
         "dns.question.name": "czbaoyu.com",
         "dns.question.type": "AAAA",
         "dns.response_code": "NOERROR",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "coredns.log",
         "event.duration": 62860,
+        "event.kind": "event",
         "event.module": "coredns",
         "event.original": "{\"message\":\"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \\\"AAAA IN czbaoyu.com. udp 29 false 512\\\" NOERROR qr,rd,ra 100 0.00006286s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-11T07:16:34.013970788Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }",
+        "event.outcome": "success",
+        "event.type": [
+            "protocol"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "kubernetes.container.name": "coredns",
@@ -143,7 +175,11 @@
         "log.level": "INFO",
         "log.offset": 1081,
         "message": "2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \"AAAA IN czbaoyu.com. udp 29 false 512\" NOERROR qr,rd,ra 100 0.00006286s\n",
+        "network.protocol": "dns",
         "network.transport": "udp",
+        "related.ip": [
+            "::1"
+        ],
         "service.type": "coredns",
         "source.address": "::1",
         "source.ip": "::1",

diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json
@@ -24,15 +24,27 @@
         "dns.question.name": "httpbin.org.cluster.local",
         "dns.question.type": "A",
         "dns.response_code": "NXDOMAIN",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "coredns.log",
         "event.duration": 102078,
+        "event.kind": "event",
         "event.module": "coredns",
+        "event.outcome": "failure",
+        "event.type": [
+            "protocol"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "INFO",
         "log.offset": 0,
         "message": "2019-03-06T08:55:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s",
+        "network.protocol": "dns",
         "network.transport": "udp",
+        "related.ip": [
+            "172.17.0.4"
+        ],
         "service.type": "coredns",
         "source.address": "172.17.0.4",
         "source.ip": "172.17.0.4",
@@ -66,15 +78,27 @@
         "dns.question.name": "www.yahoo.com",
         "dns.question.type": "A",
         "dns.response_code": "NOERROR",
+        "event.category": [
+            "network"
+        ],
         "event.dataset": "coredns.log",
         "event.duration": 20948545,
+        "event.kind": "event",
         "event.module": "coredns",
+        "event.outcome": "success",
+        "event.type": [
+            "protocol"
+        ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "INFO",
         "log.offset": 143,
         "message": "2019-03-18T22:13:36.289-07:00 [INFO] [::1]:57413 - 14639 \"A IN www.yahoo.com. udp 42 false 4096\" NOERROR qr,rd,ra 188 0.020948545s",
+        "network.protocol": "dns",
         "network.transport": "udp",
+        "related.ip": [
+            "::1"
+        ],
         "service.type": "coredns",
         "source.address": "::1",
         "source.ip": "::1",