From ec2726c975733f6a2fc6b7d96f13a892db11d28e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 29 Aug 2019 18:13:08 -0400 Subject: [PATCH 1/3] Convert coredns ingest pipeline to YAML --- .../coredns/log/ingest/pipeline-entry.json | 101 ------------------ .../coredns/log/ingest/pipeline-entry.yml | 60 +++++++++++ .../coredns/log/ingest/pipeline-json.json | 49 --------- .../coredns/log/ingest/pipeline-json.yml | 33 ++++++ .../log/ingest/pipeline-plaintext.json | 23 ---- .../coredns/log/ingest/pipeline-plaintext.yml | 17 +++ .../filebeat/module/coredns/log/manifest.yml | 8 +- 7 files changed, 114 insertions(+), 177 deletions(-) delete mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json create mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml delete mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json create mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml delete mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json create mode 100644 x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json deleted file mode 100644 index bea5c11f4ed..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "description": "Pipeline for normalizing Kubernetes coredns logs", - "processors": [ - { - "pipeline": { - "if": "ctx.message.charAt(0) == (char)(\"{\")", - "name": "{< IngestPipeline "pipeline-json" >}" - } - }, - { - "pipeline": { - "if": "ctx.message.charAt(0) != (char)(\"{\")", - "name": "{< IngestPipeline "pipeline-plaintext" >}" - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; ctx.remove('timestamp');", - "ignore_failure" : true - } - }, - { - "script": { - "lang": "painless", - "source": "ctx['source'] = new HashMap(); if (ctx.temp.source.charAt(0) == (char)(\"[\")) { def p = ctx.temp.source.indexOf (']'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(1, p); ctx.source.port = ctx.temp.source.substring(p+2, l);} else { def p = ctx.temp.source.indexOf (':'); def l = ctx.temp.source.length(); ctx.source.address = ctx.temp.source.substring(0, p); ctx.source.port = ctx.temp.source.substring(p+1, l);} ctx.remove('temp');", - "if": "ctx.temp?.source != null" - } - }, - { - "set": { - "field": "source.ip", - "value": "{{source.address}}", - "if": "ctx.source?.address != null" - } - }, - { - "convert" : { - "field" : "source.port", - "type": "integer" - } - }, - { - "convert" : { - "field" : "coredns.duration", - "type": "double" - } - }, - { - "convert" : { - "field" : "coredns.query.size", - "type": "long" - } - }, - { - "convert" : { - "field" : "coredns.response.size", - "type": "long" - } - }, - { - "convert" : { - "field" : "coredns.dnssec_ok", - "type": "boolean" - } - }, - { - "uppercase": { - "field": "coredns.response.flags" - } - }, - { - "split": { - "field": "coredns.response.flags", - "separator": "," - } - }, - { - "script": { - "lang": "painless", - "source": "ctx.event.duration = Math.round(ctx.coredns.duration * params.scale)", - "params": { - "scale": 1000000000 - }, - "if": "ctx.coredns?.duration != null" - } - }, - { - "remove": { - "field": "coredns.duration", - "ignore_missing": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml new file mode 100644 index 00000000000..95246eb1eb9 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -0,0 +1,60 @@ +--- +description: Pipeline for normalizing Kubernetes coredns logs +processors: + - pipeline: + if: ctx.message.charAt(0) == (char)("{") + name: '{< IngestPipeline "pipeline-json" >}' + - pipeline: + if: ctx.message.charAt(0) != (char)("{") + name: '{< IngestPipeline "pipeline-plaintext" >}' + - script: + lang: painless + source: ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; + ctx.remove('timestamp'); + ignore_failure: true + - script: + lang: painless + source: ctx['source'] = new HashMap(); if (ctx.temp.source.charAt(0) == (char)("[")) + { def p = ctx.temp.source.indexOf (']'); def l = ctx.temp.source.length(); ctx.source.address + = ctx.temp.source.substring(1, p); ctx.source.port = ctx.temp.source.substring(p+2, + l);} else { def p = ctx.temp.source.indexOf (':'); def l = ctx.temp.source.length(); + ctx.source.address = ctx.temp.source.substring(0, p); ctx.source.port = ctx.temp.source.substring(p+1, + l);} ctx.remove('temp'); + if: ctx.temp?.source != null + - set: + field: source.ip + value: "{{source.address}}" + if: ctx.source?.address != null + - convert: + field: source.port + type: integer + - convert: + field: coredns.duration + type: double + - convert: + field: coredns.query.size + type: long + - convert: + field: coredns.response.size + type: long + - convert: + field: coredns.dnssec_ok + type: boolean + - uppercase: + field: coredns.response.flags + - split: + field: coredns.response.flags + separator: "," + - script: + lang: painless + source: ctx.event.duration = Math.round(ctx.coredns.duration * params.scale) + params: + scale: 1000000000 + if: ctx.coredns?.duration != null + - remove: + field: coredns.duration + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json deleted file mode 100644 index 5825fb5544b..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "description": "Pipeline for dissecting message field in JSON logs", - "processors": [ - { - "json" : { - "field" : "message", - "target_field" : "json" - } - }, - { - "dissect": { - "field": "json.message", - "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" - } - }, - { - "remove": { - "field": ["message"], - "ignore_failure" : true - } - }, - { - "rename": { - "field": "json.message", - "target_field": "message", - "ignore_failure" : true - } - }, - { - "rename": { - "field": "json.kubernetes", - "target_field": "kubernetes", - "ignore_failure" : true - } - }, - { - "remove": { - "field": ["json", "bufsize"], - "ignore_failure" : true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml new file mode 100644 index 00000000000..a2f1fb884f1 --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml @@ -0,0 +1,33 @@ +--- +description: Pipeline for dissecting message field in JSON logs +processors: + - json: + field: message + target_field: json + - dissect: + field: json.message + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} "%{coredns.query.type} + %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{bufsize}" %{coredns.response.code} %{coredns.response.flags} + %{coredns.response.size} %{coredns.duration}s' + - remove: + field: + - message + ignore_failure: true + - rename: + field: json.message + target_field: message + ignore_failure: true + - rename: + field: json.kubernetes + target_field: kubernetes + ignore_failure: true + - remove: + field: + - json + - bufsize + ignore_failure: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json deleted file mode 100644 index f1b0c1d26ea..00000000000 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "description": "Pipeline for dissecting text in plaintext logs", - "processors": [ - { - "dissect": { - "field": "message", - "pattern": "%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{coredns.duration}s" - } - }, - { - "remove": { - "field": ["bufsize"], - "ignore_failure" : true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml new file mode 100644 index 00000000000..a284fb7c40c --- /dev/null +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml @@ -0,0 +1,17 @@ +--- +description: Pipeline for dissecting text in plaintext logs +processors: + - dissect: + field: message + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} "%{coredns.query.type} + %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{bufsize}" %{coredns.response.code} %{coredns.response.flags} + %{coredns.response.size} %{coredns.duration}s' + - remove: + field: + - bufsize + ignore_failure: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/coredns/log/manifest.yml b/x-pack/filebeat/module/coredns/log/manifest.yml index 34997379524..e41e8c30387 100644 --- a/x-pack/filebeat/module/coredns/log/manifest.yml +++ b/x-pack/filebeat/module/coredns/log/manifest.yml @@ -8,8 +8,8 @@ var: - name: tags default: [coredns] -ingest_pipeline: - - ingest/pipeline-entry.json - - ingest/pipeline-json.json - - ingest/pipeline-plaintext.json +ingest_pipeline: + - ingest/pipeline-entry.yml + - ingest/pipeline-json.yml + - ingest/pipeline-plaintext.yml input: config/coredns.yml From 23e65a1975b61653abca7f255c6798c4386c4731 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 30 Aug 2019 13:00:47 -0400 Subject: [PATCH 2/3] Update pipelines for ECS DNS This sets the ECS DNS fields. It does not remove the coredns.* fields to avoid introducing a breaking change. Relates #13320 --- CHANGELOG.next.asciidoc | 1 + .../coredns/log/ingest/pipeline-entry.yml | 67 +++++++++++++++---- .../coredns/log/ingest/pipeline-json.yml | 19 +++--- .../coredns/log/ingest/pipeline-plaintext.yml | 12 ++-- .../log/test/coredns-json.log-expected.json | 33 +++++++++ .../log/test/coredns.log-expected.json | 20 ++++++ 6 files changed, 122 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2e2f260c5b2..6a406dda3a6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -299,6 +299,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add container image in Kubernetes metadata {pull}13356[13356] {issue}12688[12688] - Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286] +- Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505] *Heartbeat* diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index 95246eb1eb9..4879030f2f2 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -1,5 +1,5 @@ --- -description: Pipeline for normalizing Kubernetes coredns logs +description: Pipeline for normalizing Kubernetes CoreDNS logs. processors: - pipeline: if: ctx.message.charAt(0) == (char)("{") @@ -9,18 +9,28 @@ processors: name: '{< IngestPipeline "pipeline-plaintext" >}' - script: lang: painless - source: ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; + source: > + ctx.event.created = ctx['@timestamp']; + ctx['@timestamp'] = ctx['timestamp']; ctx.remove('timestamp'); ignore_failure: true - script: lang: painless - source: ctx['source'] = new HashMap(); if (ctx.temp.source.charAt(0) == (char)("[")) - { def p = ctx.temp.source.indexOf (']'); def l = ctx.temp.source.length(); ctx.source.address - = ctx.temp.source.substring(1, p); ctx.source.port = ctx.temp.source.substring(p+2, - l);} else { def p = ctx.temp.source.indexOf (':'); def l = ctx.temp.source.length(); - ctx.source.address = ctx.temp.source.substring(0, p); ctx.source.port = ctx.temp.source.substring(p+1, - l);} ctx.remove('temp'); if: ctx.temp?.source != null + source: > + ctx['source'] = new HashMap(); + if (ctx.temp.source.charAt(0) == (char)("[")) { + def p = ctx.temp.source.indexOf (']'); + def l = ctx.temp.source.length(); + ctx.source.address = ctx.temp.source.substring(1, p); + ctx.source.port = ctx.temp.source.substring(p+2, l); + } else { + def p = ctx.temp.source.indexOf(':'); + def l = ctx.temp.source.length(); + ctx.source.address = ctx.temp.source.substring(0, p); + ctx.source.port = ctx.temp.source.substring(p+1, l); + } + ctx.remove('temp'); - set: field: source.ip value: "{{source.address}}" @@ -41,19 +51,52 @@ processors: field: coredns.dnssec_ok type: boolean - uppercase: - field: coredns.response.flags + field: dns.header_flags - split: - field: coredns.response.flags + field: dns.header_flags separator: "," + - append: + if: ctx.coredns?.dnssec_ok + field: dns.header_flags + value: DO - script: lang: painless - source: ctx.event.duration = Math.round(ctx.coredns.duration * params.scale) + source: ctx.event.duration = Math.round(ctx.coredns.duration * params.scale); params: scale: 1000000000 if: ctx.coredns?.duration != null - remove: - field: coredns.duration + field: + - coredns.duration ignore_missing: true + # The following copies values from dns namespace (ECS) to the coredns + # namespace to avoid introducing breaking change. This should be removed + # for 8.0.0. Additionally coredns.dnssec_ok can be removed. + - set: + if: ctx.dns?.id != null + field: coredns.id + value: '{{dns.id}}' + - set: + if: ctx.dns?.question?.class != null + field: coredns.query.class + value: '{{dns.question.class}}' + - set: + if: ctx.dns?.question?.name != null + field: coredns.query.name + value: '{{dns.question.name}}' + - set: + if: ctx.dns?.question?.type != null + field: coredns.query.type + value: '{{dns.question.type}}' + - set: + if: ctx.dns?.response_code != null + field: coredns.response.code + value: '{{dns.response_code}}' + - script: + if: ctx.dns?.header_flags != null + lang: painless + source: > + ctx.coredns.response.flags = ctx.dns.header_flags; on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml index a2f1fb884f1..0b89ab35252 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml @@ -1,19 +1,19 @@ --- -description: Pipeline for dissecting message field in JSON logs +description: Pipeline for dissecting CoreDNS JSON logs. processors: - - json: + - rename: field: message + target_field: event.original + ignore_failure: true + - json: + field: event.original target_field: json - dissect: field: json.message - pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} "%{coredns.query.type} - %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} - %{coredns.dnssec_ok} %{bufsize}" %{coredns.response.code} %{coredns.response.flags} + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type} + %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags} %{coredns.response.size} %{coredns.duration}s' - - remove: - field: - - message - ignore_failure: true - rename: field: json.message target_field: message @@ -25,7 +25,6 @@ processors: - remove: field: - json - - bufsize ignore_failure: true on_failure: - set: diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml index a284fb7c40c..fcec1fffdc1 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml @@ -1,16 +1,12 @@ --- -description: Pipeline for dissecting text in plaintext logs +description: Pipeline for dissecting CoreDNS plaintext logs. processors: - dissect: field: message - pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{coredns.id} "%{coredns.query.type} - %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} - %{coredns.dnssec_ok} %{bufsize}" %{coredns.response.code} %{coredns.response.flags} + pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type} + %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size} + %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags} %{coredns.response.size} %{coredns.duration}s' - - remove: - field: - - bufsize - ignore_failure: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json index 12e84895078..ff394d4d481 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -14,9 +14,20 @@ "RA" ], "coredns.response.size": 136, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "21583", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org.cluster.local.", + "dns.question.type": "A", + "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", "event.duration": 102078, "event.module": "coredns", + "event.original": "{\"message\":\"2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \\\"A IN httpbin.org.cluster.local. udp 43 false 512\\\" NXDOMAIN qr,rd,ra 136 0.000102078s\", \"stream\": \"stdout\", \"time\": \"2019-02-12T00:27:28.903433597Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -54,9 +65,20 @@ "RA" ], "coredns.response.size": 83, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "6966", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org.", + "dns.question.type": "A", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 82083, "event.module": "coredns", + "event.original": "{\"message\":\"2019-03-19T02:57:23.213Z [INFO] 172.17.0.9:37723 - 6966 \\\"A IN httpbin.org. udp 29 false 512\\\" NOERROR qr,rd,ra 83 0.000082083s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-19T02:57:23.214583742Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", @@ -94,9 +116,20 @@ "RA" ], "coredns.response.size": 100, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "62762", + "dns.question.class": "IN", + "dns.question.name": "czbaoyu.com.", + "dns.question.type": "AAAA", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 62860, "event.module": "coredns", + "event.original": "{\"message\":\"2019-03-11T07:16:34.013Z [INFO] [::1]:37915 - 62762 \\\"AAAA IN czbaoyu.com. udp 29 false 512\\\" NOERROR qr,rd,ra 100 0.00006286s\\n\",\"stream\":\"stdout\",\"time\":\"2019-03-11T07:16:34.013970788Z\", \"kubernetes\": { \"container\": { \"name\": \"coredns\" }, \"node\": { \"name\": \"minikube\" }, \"pod\": { \"uid\": \"d57d545e-2a9d-11e9-995f-08002730e0dc\", \"name\": \"coredns-86c58d9df4-jwhsg\" }, \"namespace\": \"kube-system\", \"replicaset\": { \"name\": \"coredns-86c58d9df4\" }, \"labels\": { \"pod-template-hash\": \"86c58d9df4\", \"k8s-app\": \"kube-dns\" } } }", "fileset.name": "log", "input.type": "log", "kubernetes.container.name": "coredns", diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json index 14a2cfba2e7..c579aa1d634 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -14,6 +14,16 @@ "RA" ], "coredns.response.size": 136, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "21583", + "dns.question.class": "IN", + "dns.question.name": "httpbin.org.cluster.local.", + "dns.question.type": "A", + "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", "event.duration": 102078, "event.module": "coredns", @@ -46,6 +56,16 @@ "RA" ], "coredns.response.size": 188, + "dns.header_flags": [ + "QR", + "RD", + "RA" + ], + "dns.id": "14639", + "dns.question.class": "IN", + "dns.question.name": "www.yahoo.com.", + "dns.question.type": "A", + "dns.response_code": "NOERROR", "event.dataset": "coredns.log", "event.duration": 20948545, "event.module": "coredns", From 3402b8d7eb9e1636c1d638b25781a2a60c99e3aa Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 5 Sep 2019 14:34:22 -0400 Subject: [PATCH 3/3] Right trim trailing dots in dns.question.name --- .../module/coredns/log/ingest/pipeline-entry.yml | 10 ++++++++++ .../coredns/log/test/coredns-json.log-expected.json | 6 +++--- .../module/coredns/log/test/coredns.log-expected.json | 4 ++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index 4879030f2f2..8f55838b21e 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -97,6 +97,16 @@ processors: lang: painless source: > ctx.coredns.response.flags = ctx.dns.header_flags; + # Right trim the trailing dot from domain names. + - script: + if: ctx.dns?.question?.name != null + lang: painless + source: > + def q = ctx.dns.question.name; + def end = q.length() - 1; + if (q.charAt(end) == (char) '.') { + ctx.dns.question.name = q.substring(0, end); + } on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json index ff394d4d481..637f8cb0b4d 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns-json.log-expected.json @@ -21,7 +21,7 @@ ], "dns.id": "21583", "dns.question.class": "IN", - "dns.question.name": "httpbin.org.cluster.local.", + "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", @@ -72,7 +72,7 @@ ], "dns.id": "6966", "dns.question.class": "IN", - "dns.question.name": "httpbin.org.", + "dns.question.name": "httpbin.org", "dns.question.type": "A", "dns.response_code": "NOERROR", "event.dataset": "coredns.log", @@ -123,7 +123,7 @@ ], "dns.id": "62762", "dns.question.class": "IN", - "dns.question.name": "czbaoyu.com.", + "dns.question.name": "czbaoyu.com", "dns.question.type": "AAAA", "dns.response_code": "NOERROR", "event.dataset": "coredns.log", diff --git a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json index c579aa1d634..ba3191a9e17 100644 --- a/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json +++ b/x-pack/filebeat/module/coredns/log/test/coredns.log-expected.json @@ -21,7 +21,7 @@ ], "dns.id": "21583", "dns.question.class": "IN", - "dns.question.name": "httpbin.org.cluster.local.", + "dns.question.name": "httpbin.org.cluster.local", "dns.question.type": "A", "dns.response_code": "NXDOMAIN", "event.dataset": "coredns.log", @@ -63,7 +63,7 @@ ], "dns.id": "14639", "dns.question.class": "IN", - "dns.question.name": "www.yahoo.com.", + "dns.question.name": "www.yahoo.com", "dns.question.type": "A", "dns.response_code": "NOERROR", "event.dataset": "coredns.log",