diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2e2f260c5b2..6a406dda3a6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -299,6 +299,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add container image in Kubernetes metadata {pull}13356[13356] {issue}12688[12688] - Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] +- Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505] *Heartbeat* diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json deleted file mode 100644 index bea5c11f4ed..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "description": "Pipeline for normalizing Kubernetes coredns logs", - "processors": [ - { - "pipeline": { - "if": "ctx.message.charAt(0) == (char)(\"{\")", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "pipeline": { - "if": "ctx.message.charAt(0) != (char)(\"{\")", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; ctx.remove('timestamp');", - "ignore_failure" : true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx['source'] = new HashMap(); if (ctx.temp.source.charAt(0) == (char)(\"[\")) { def p = ctx.temp.source.indexOf (']'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(1, p); ctx.source.port = ctx.temp.source.substring(p+2, l);} else { def p = ctx.temp.source.indexOf (':'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(0, p); ctx.source.port = ctx.temp.source.substring(p+1, l);} ctx.remove('temp');", - "if": "ctx.temp?.source != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}", - "if": "ctx.source?.address != null" - } - }, - { - "convert" : { - "field" : "source.port", - "type": "integer" - } - }, - { - "convert" : { - "field" : "coredns.duration", - "type": "double" - } - }, - { - "convert" : { - "field" : "coredns.query.size", - "type": "long" - } - }, - { - "convert" : { - "field" : "coredns.response.size", - "type": "long" - } - }, - { - "convert" : { - "field" : "coredns.dnssec_ok", - "type": "boolean" - } - }, - { - "uppercase": { - "field": "coredns.response.flags" - } - }, - { - "split": { - "field": "coredns.response.flags", - "separator": "," - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.coredns.duration * params.scale)", - "params": { - "scale": 1000000000 - }, - "if": "ctx.coredns?.duration != null" - } - }, - { - "remove": { - "field": "coredns.duration", - "ignore_missing": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml new file mode 100644 index 00000000000..8f55838b21e --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -0,0 +1,113 @@ +--- +description: Pipeline for normalizing Kubernetes CoreDNS logs. +processors: + - pipeline: + if: ctx.message.charAt(0) == (char)("{") + name: '{< IngestPipeline "pipeline-json" >}' + - pipeline: + if: ctx.message.charAt(0) != (char)("{") + name: '{< IngestPipeline "pipeline-plaintext" >}' + - script: + lang: painless + source: > + ctx.event.created = ctx['@timestamp']; + ctx['@timestamp'] = ctx['timestamp']; + ctx.remove('timestamp'); + ignore_failure: true + - script: + lang: painless + if: ctx.temp?.source != null + source: > + ctx['source'] = new HashMap(); + if (ctx.temp.source.charAt(0) == (char)("[")) { + def p = ctx.temp.source.indexOf (']'); + def l = ctx.temp.source.length(); + ctx.source.address = ctx.temp.source.substring(1, p); + ctx.source.port = ctx.temp.source.substring(p+2, l); + } else { + def p = ctx.temp.source.indexOf(':'); + def l = ctx.temp.source.length(); + ctx.source.address = ctx.temp.source.substring(0, p); + ctx.source.port = ctx.temp.source.substring(p+1, l); + } + ctx.remove('temp'); + - set: + field: source.ip + value: "{{source.address}}" + if: ctx.source?.address != null + - convert: + field: source.port + type: integer + - convert: + field: coredns.duration + type: double + - convert: + field: coredns.query.size + type: long + - convert: + field: coredns.response.size + type: long + - convert: + field: coredns.dnssec_ok + type: boolean + - uppercase: + field: dns.header_flags + - split: + field: dns.header_flags + separator: "," + - append: + if: ctx.coredns?.dnssec_ok + field: dns.header_flags + value: DO + - script: + lang: painless + source: ctx.event.duration = Math.round(ctx.coredns.duration * params.scale); + params: + scale: 1000000000 + if: ctx.coredns?.duration != null + - remove: + field: + - coredns.duration + ignore_missing: true + # The following copies values from dns namespace (ECS) to the coredns + # namespace to avoid introducing breaking change. This should be removed + # for 8.0.0. Additionally coredns.dnssec_ok can be removed. + - set: + if: ctx.dns?.id != null + field: coredns.id + value: '{{dns.id}}' + - set: + if: ctx.dns?.question?.class != null + field: coredns.query.class + value: '{{dns.question.class}}' + - set: + if: ctx.dns?.question?.name != null + field: coredns.query.name + value: '{{dns.question.name}}' + - set: + if: ctx.dns?.question?.type != null + field: coredns.query.type + value: '{{dns.question.type}}' + - set: + if: ctx.dns?.response_code != null + field: coredns.response.code + value: '{{dns.response_code}}' + - script: + if: ctx.dns?.header_flags != null + lang: painless + source: > + ctx.coredns.response.flags = ctx.dns.header_flags; + # Right trim the trailing dot from domain names. + - script: + if: ctx.dns?.question?.name != null + lang: painless + source: > + def q = ctx.dns.question.name; + def end = q.length() - 1; + if (q.charAt(end) == (char) '.') { + ctx.dns.question.name = q.substring(0, end); + } +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json deleted file mode 100644 index 5825fb5544b..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "description": "Pipeline for dissecting message field in JSON logs", - "processors": [ - { - "json" : { - "field" : "message", - "target_field" : "json" - } - }, - { - "dissect": { - "field": "json.message", - "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" - } - }, - { - "remove": { - "field": ["message"], - "ignore_failure" : true - } - }, - { - "rename": { - "field": "json.message", - "target_field": "message", - "ignore_failure" : true - } - }, - { - "rename": { - "field": "json.kubernetes", - "target_field": "kubernetes", - "ignore_failure" : true - } - }, - { - "remove": { - "field": ["json", "bufsize"], - "ignore_failure" : true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml new file mode 100644 index 00000000000..0b89ab35252 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml @@ -0,0 +1,32 @@ +--- +description: Pipeline for dissecting CoreDNS JSON logs. +processors: + - rename: + field: message + target_field: event.original + ignore_failure: true + - json: + field: event.original + target_field: json + - dissect: + field: json.message + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type} + %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags} + %{coredns.response.size} %{coredns.duration}s' + - rename: + field: json.message + target_field: message + ignore_failure: true + - rename: + field: json.kubernetes + target_field: kubernetes + ignore_failure: true + - remove: + field: + - json + ignore_failure: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json deleted file mode 100644 index f1b0c1d26ea..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "description": "Pipeline for dissecting text in plaintext logs", - "processors": [ - { - "dissect": { - "field": "message", - "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" - } - }, - { - "remove": { - "field": ["bufsize"], - "ignore_failure" : true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..fcec1fffdc1 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml @@ -0,0 +1,13 @@ +--- +description: Pipeline for dissecting CoreDNS plaintext logs. +processors: + - dissect: + field: message + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type} + %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags} + %{coredns.response.size} %{coredns.duration}s' +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/manifest.yml b/x-pack/filebeat/module/coredns/log/manifest.yml index 34997379524..e41e8c30387 100644 --- a/x-pack/filebeat/module/coredns/log/manifest.yml +++ b/x-pack/filebeat/module/coredns/log/manifest.yml @@ -8,8 +8,8 @@ var: - name: tags default: [coredns] -ingest_pipeline: - - ingest/pipeline-entry.json - - ingest/pipeline-json.json - - ingest/pipeline-plaintext.json +ingest_pipeline: + - ingest/pipeline-entry.yml + - ingest/pipeline-json.yml + - ingest/pipeline-plaintext.yml input: config/coredns.yml diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json index 12e84895078..637f8cb0b4d 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -14,9 +14,20 @@ "RA" ], "coredns.response.size": 136, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "21583", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org.cluster.local", + "dns.question.type": "A", + "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", "event.duration": 102078, "event.module": "coredns", + "event.original": "{\"message\":\"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \\\"A IN httpbin.org.cluster.local. udp 43 false 512\\\" NXDOMAIN qr,rd,ra 136 0.000102078s\", \"stream\": \"stdout\", \"time\": \"2019-02-12T00:27:28.903433597Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -54,9 +65,20 @@ "RA" ], "coredns.response.size": 83, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "6966", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org", + "dns.question.type": "A", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 82083, "event.module": "coredns", + "event.original": "{\"message\":\"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \\\"A IN httpbin.org. udp 29 false 512\\\" NOERROR qr,rd,ra 83 0.000082083s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-19T02:57:23.214583742Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -94,9 +116,20 @@ "RA" ], "coredns.response.size": 100, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "62762", + "dns.question.class": "IN", + "dns.question.name": "czbaoyu.com", + "dns.question.type": "AAAA", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 62860, "event.module": "coredns", + "event.original": "{\"message\":\"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \\\"AAAA IN czbaoyu.com. udp 29 false 512\\\" NOERROR qr,rd,ra 100 0.00006286s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-11T07:16:34.013970788Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json index 14a2cfba2e7..ba3191a9e17 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -14,6 +14,16 @@ "RA" ], "coredns.response.size": 136, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "21583", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org.cluster.local", + "dns.question.type": "A", + "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", "event.duration": 102078, "event.module": "coredns", @@ -46,6 +56,16 @@ "RA" ], "coredns.response.size": 188, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "14639", + "dns.question.class": "IN", + "dns.question.name": "www.yahoo.com", + "dns.question.type": "A", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 20948545, "event.module": "coredns",