[Filebeat] Update CoreDNS pipelines for ECS DNS (#13505)

This sets the ECS DNS fields. It does not remove the coredns.* fields to avoid introducing
a breaking change.

* Convert coredns ingest pipeline to YAML
* Update pipelines for ECS DNS
* Right trim trailing dots in dns.question.name

Relates #13320

CHANGELOG.next.asciidoc

@@ -302,6 +302,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add container image in Kubernetes metadata {pull}13356[13356] {issue}12688[12688]
 - Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304]
 - Add module for ingesting Cisco FTD logs over syslog. {pull}13286[13286]
+- Update CoreDNS module to populate ECS DNS fields. {issue}13320[13320] {pull}13505[13505]
 
 *Heartbeat*
 

x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml

@@ -0,0 +1,113 @@
+---
+description: Pipeline for normalizing Kubernetes CoreDNS logs.
+processors:
+  - pipeline:
+      if: ctx.message.charAt(0) == (char)("{")
+      name: '{< IngestPipeline "pipeline-json" >}'
+  - pipeline:
+      if: ctx.message.charAt(0) != (char)("{")
+      name: '{< IngestPipeline "pipeline-plaintext" >}'
+  - script:
+      lang: painless
+      source: >
+        ctx.event.created = ctx['@timestamp'];
+        ctx['@timestamp'] = ctx['timestamp'];
+        ctx.remove('timestamp');
+      ignore_failure: true
+  - script:
+      lang: painless
+      if: ctx.temp?.source != null
+      source: >
+        ctx['source'] = new HashMap();
+        if (ctx.temp.source.charAt(0) == (char)("[")) {
+            def p = ctx.temp.source.indexOf (']');
+            def l = ctx.temp.source.length();
+            ctx.source.address = ctx.temp.source.substring(1, p);
+            ctx.source.port = ctx.temp.source.substring(p+2, l);
+        } else {
+            def p = ctx.temp.source.indexOf(':');
+            def l = ctx.temp.source.length();
+            ctx.source.address = ctx.temp.source.substring(0, p);
+            ctx.source.port = ctx.temp.source.substring(p+1, l);
+        }
+        ctx.remove('temp');
+  - set:
+      field: source.ip
+      value: "{{source.address}}"
+      if: ctx.source?.address != null
+  - convert:
+      field: source.port
+      type: integer
+  - convert:
+      field: coredns.duration
+      type: double
+  - convert:
+      field: coredns.query.size
+      type: long
+  - convert:
+      field: coredns.response.size
+      type: long
+  - convert:
+      field: coredns.dnssec_ok
+      type: boolean
+  - uppercase:
+      field: dns.header_flags
+  - split:
+      field: dns.header_flags
+      separator: ","
+  - append:
+      if: ctx.coredns?.dnssec_ok
+      field: dns.header_flags
+      value: DO
+  - script:
+      lang: painless
+      source: ctx.event.duration = Math.round(ctx.coredns.duration * params.scale);
+      params:
+        scale: 1000000000
+      if: ctx.coredns?.duration != null
+  - remove:
+      field:
+        - coredns.duration
+      ignore_missing: true
+  # The following copies values from dns namespace (ECS) to the coredns
+  # namespace to avoid introducing breaking change. This should be removed
+  # for 8.0.0. Additionally coredns.dnssec_ok can be removed.
+  - set:
+      if: ctx.dns?.id != null
+      field: coredns.id
+      value: '{{dns.id}}'
+  - set:
+      if: ctx.dns?.question?.class != null
+      field: coredns.query.class
+      value: '{{dns.question.class}}'
+  - set:
+      if: ctx.dns?.question?.name != null
+      field: coredns.query.name
+      value: '{{dns.question.name}}'
+  - set:
+      if: ctx.dns?.question?.type != null
+      field: coredns.query.type
+      value: '{{dns.question.type}}'
+  - set:
+      if: ctx.dns?.response_code != null
+      field: coredns.response.code
+      value: '{{dns.response_code}}'
+  - script:
+      if: ctx.dns?.header_flags != null
+      lang: painless
+      source: >
+        ctx.coredns.response.flags = ctx.dns.header_flags;
+  # Right trim the trailing dot from domain names.
+  - script:
+      if: ctx.dns?.question?.name != null
+      lang: painless
+      source: >
+        def q = ctx.dns.question.name;
+        def end = q.length() - 1;
+        if (q.charAt(end) == (char) '.') {
+            ctx.dns.question.name = q.substring(0, end);
+        }
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/coredns/log/ingest/pipeline-json.yml

@@ -0,0 +1,32 @@
+---
+description: Pipeline for dissecting CoreDNS JSON logs.
+processors:
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_failure: true
+  - json:
+      field: event.original
+      target_field: json
+  - dissect:
+      field: json.message
+      pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type}
+      %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size}
+      %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags}
+      %{coredns.response.size} %{coredns.duration}s'
+  - rename:
+      field: json.message
+      target_field: message
+      ignore_failure: true
+  - rename:
+      field: json.kubernetes
+      target_field: kubernetes
+      ignore_failure: true
+  - remove:
+      field:
+        - json
+      ignore_failure: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/coredns/log/ingest/pipeline-plaintext.yml

@@ -0,0 +1,13 @@
+---
+description: Pipeline for dissecting CoreDNS plaintext logs.
+processors:
+  - dissect:
+      field: message
+      pattern: '%{timestamp} [%{log.level}] %{temp.source} - %{dns.id} "%{dns.question.type}
+      %{dns.question.class} %{dns.question.name} %{network.transport} %{coredns.query.size}
+      %{coredns.dnssec_ok} %{?bufsize}" %{dns.response_code} %{dns.header_flags}
+      %{coredns.response.size} %{coredns.duration}s'
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"

x-pack/filebeat/module/coredns/log/manifest.yml

@@ -8,8 +8,8 @@ var:
   - name: tags
     default: [coredns]
 
-ingest_pipeline: 
-  - ingest/pipeline-entry.json
-  - ingest/pipeline-json.json
-  - ingest/pipeline-plaintext.json
+ingest_pipeline:
+  - ingest/pipeline-entry.yml
+  - ingest/pipeline-json.yml
+  - ingest/pipeline-plaintext.yml
 input: config/coredns.yml