[Filebeat][zeek] Add new x509 fields to zeek

CHANGELOG.next.asciidoc

@@ -548,6 +548,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Return error when log harvester tries to open a named pipe. {issue}18682[18682] {pull}20450[20450]
 - Avoid goroutine leaks in Filebeat readers. {issue}19193[19193] {pull}20455[20455]
 - Convert httpjson to v2 input {pull}20226[20226]
+- Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
 
 *Heartbeat*
 

x-pack/filebeat/module/zeek/x509/ingest/pipeline.yml

@@ -17,42 +17,133 @@ processors:
       field: event.id
       value: '{{zeek.session_id}}'
       if: ctx.zeek.session_id != null
+  - set:
+      field: file.x509.signature_algorithm
+      value: '{{zeek.x509.certificate.signature_algorithm}}'
+      ignore_empty_value: true
+  - script:
+      lang: painless
+      params:
+        "md2WithRSAEncryption": MD2-RSA
+        "md5WithRSAEncryption": MD5-RSA
+        "sha-1WithRSAEncryption": SHA1-RSA
+        "sha256WithRSAEncryption": SHA256-RSA
+        "sha384WithRSAEncryption": SHA384-RSA
+        "sha512WithRSAEncryption": SHA512-RSA
+        "dsaWithSha1": DSA-SHA1
+        "dsaWithSha256": DSA-SHA256
+        "ecdsa-with-SHA1": ECDSA-SHA1
+        "ecdsa-with-SHA256": ECDSA-SHA256
+        "ecdsa-with-SHA384": ECDSA-SHA384
+        "ecdsa-with-SHA512": ECDSA-SHA512
+        "id-Ed25519": Ed25519
+      source: |
+        String algo = params.get(ctx.file.x509.signature_algorithm);
+        if (algo != null) {
+          ctx.file.x509.signature_algorithm = algo;
+        }
+      if: ctx?.file?.x509?.signature_algorithm != null
+  - set:
+      field: file.x509.public_key_algorithm
+      value: '{{zeek.x509.certificate.key.algorithm}}'
+      ignore_empty_value: true
+  - convert:
+      field: zeek.x509.certificate.key.length
+      target_field: file.x509.public_key_size
+      type: long
+      ignore_missing: true
   - dot_expander:
       field: certificate.exponent
       path: zeek.x509
+  - convert:
+      field: zeek.x509.certificate.exponent
+      target_field: file.x509.public_key_exponent
+      type: long
+      ignore_missing: true
   - dot_expander:
       field: certificate.serial
       path: zeek.x509
+  - set:
+      field: file.x509.serial_number
+      value: '{{zeek.x509.certificate.serial}}'
+      ignore_empty_value: true
   - dot_expander:
       field: certificate.version
       path: zeek.x509
+  - set:
+      field: file.x509.version_number
+      value: '{{zeek.x509.certificate.version}}'
+      ignore_empty_value: true
   - dot_expander:
       field: san.dns
       path: zeek.x509
+  - foreach:
+      field: zeek.x509.san.dns
+      ignore_missing: true
+      processor:
+        append:
+          field: file.x509.alternative_names
+          value: '{{_ingest._value}}'
   - dot_expander:
       field: san.uri
       path: zeek.x509
+  - foreach:
+      field: zeek.x509.san.uri
+      ignore_missing: true
+      processor:
+        append:
+          field: file.x509.alternative_names
+          value: '{{_ingest._value}}'
   - dot_expander:
       field: san.email
       path: zeek.x509
+  - foreach:
+      field: zeek.x509.san.email
+      ignore_missing: true
+      processor:
+        append:
+          field: file.x509.alternative_names
+          value: '{{_ingest._value}}'
   - dot_expander:
       field: san.ip
       path: zeek.x509
+  - foreach:
+      field: zeek.x509.san.ip
+      ignore_missing: true
+      processor:
+        append:
+          field: file.x509.alternative_names
+          value: '{{_ingest._value}}'
   - dot_expander:
       field: san.other_fields
       path: zeek.x509
+  - foreach:
+      field: zeek.x509.san.other_fields
+      ignore_missing: true
+      processor:
+        append:
+          field: file.x509.alternative_names
+          value: '{{_ingest._value}}'
   - date:
       field: zeek.x509.certificate.valid.from
       target_field: zeek.x509.certificate.valid.from
       formats:
         - UNIX
       if: ctx.zeek.x509.certificate?.valid?.from != null
+  - set:
+      field: file.x509.not_before
+      value: '{{zeek.x509.certificate.valid.from}}'
+      ignore_empty_value: true
   - date:
       field: zeek.x509.certificate.valid.until
       target_field: zeek.x509.certificate.valid.until
       formats:
         - UNIX
       if: ctx.zeek.x509.certificate?.valid?.until != null
+  - set:
+      field: file.x509.not_after
+      value: '{{zeek.x509.certificate.valid.until}}'
+      ignore_empty_value: true
   - gsub:
       field: zeek.x509.certificate.iss
       pattern: \\,
@@ -71,26 +162,50 @@ processors:
       field: zeek.x509.certificate.issuer.C
       target_field: zeek.x509.certificate.issuer.country
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.country
+      value: '{{zeek.x509.certificate.issuer.country}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.issuer.CN
       target_field: zeek.x509.certificate.issuer.common_name
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.common_name
+      value: '{{zeek.x509.certificate.issuer.common_name}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.issuer.L
       target_field: zeek.x509.certificate.issuer.locality
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.locality
+      value: '{{zeek.x509.certificate.issuer.locality}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.issuer.O
       target_field: zeek.x509.certificate.issuer.organization
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.organization
+      value: '{{zeek.x509.certificate.issuer.organization}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.issuer.OU
       target_field: zeek.x509.certificate.issuer.organizational_unit
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.organizational_unit
+      value: '{{zeek.x509.certificate.issuer.organizational_unit}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.issuer.ST
       target_field: zeek.x509.certificate.issuer.state
       ignore_missing: true
+  - set:
+      field: file.x509.issuer.state_or_province
+      value: '{{zeek.x509.certificate.issuer.state}}'
+      ignore_empty_value: true
   - gsub:
       field: zeek.x509.certificate.sub
       pattern: \\,
@@ -109,27 +224,51 @@ processors:
       field: zeek.x509.certificate.subject.C
       target_field: zeek.x509.certificate.subject.country
       ignore_missing: true
+  - set:
+      field: file.x509.subject.country
+      value: '{{zeek.x509.certificate.subject.country}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.subject.CN
       target_field: zeek.x509.certificate.subject.common_name
       ignore_missing: true
+  - set:
+      field: file.x509.subject.common_name
+      value: '{{zeek.x509.certificate.subject.common_name}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.subject.L
       target_field: zeek.x509.certificate.subject.locality
       ignore_missing: true
+  - set:
+      field: file.x509.subject.locality
+      value: '{{zeek.x509.certificate.subject.locality}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.subject.O
       target_field: zeek.x509.certificate.subject.organization
       ignore_missing: true
+  - set:
+      field: file.x509.subject.organization
+      value: '{{zeek.x509.certificate.subject.organization}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.subject.OU
       target_field: zeek.x509.certificate.subject.organizational_unit
       ignore_missing: true
+  - set:
+      field: file.x509.subject.organizational_unit
+      value: '{{zeek.x509.certificate.subject.organizational_unit}}'
+      ignore_empty_value: true
   - rename:
       field: zeek.x509.certificate.subject.ST
       target_field: zeek.x509.certificate.subject.state
       ignore_missing: true
+  - set:
+      field: file.x509.subject.state_or_province
+      value: '{{zeek.x509.certificate.subject.state}}'
+      ignore_empty_value: true
 on_failure:
   - set:
       field: error.message
-      value: '{{ _ingest.on_failure_message }}'
+      value: '{{_ingest.on_failure_message}}'

x-pack/filebeat/module/zeek/x509/test/x509-json.log-expected.json

@@ -8,6 +8,95 @@
         "event.type": [
             "info"
         ],
+        "file.x509.alternative_names": [
+            "www.bing.com",
+            "dict.bing.com.cn",
+            "*.platform.bing.com",
+            "*.bing.com",
+            "bing.com",
+            "ieonline.microsoft.com",
+            "*.windowssearch.com",
+            "cn.ieonline.microsoft.com",
+            "*.origin.bing.com",
+            "*.mm.bing.net",
+            "*.api.bing.com",
+            "ecn.dev.virtualearth.net",
+            "*.cn.bing.net",
+            "*.cn.bing.com",
+            "ssl-api.bing.com",
+            "ssl-api.bing.net",
+            "*.api.bing.net",
+            "*.bingapis.com",
+            "bingsandbox.com",
+            "feedback.microsoft.com",
+            "insertmedia.bing.office.net",
+            "r.bat.bing.com",
+            "*.r.bat.bing.com",
+            "*.dict.bing.com.cn",
+            "*.dict.bing.com",
+            "*.ssl.bing.com",
+            "*.appex.bing.com",
+            "*.platform.cn.bing.com",
+            "wp.m.bing.com",
+            "*.m.bing.com",
+            "global.bing.com",
+            "windowssearch.com",
+            "search.msn.com",
+            "*.bingsandbox.com",
+            "*.api.tiles.ditu.live.com",
+            "*.ditu.live.com",
+            "*.t0.tiles.ditu.live.com",
+            "*.t1.tiles.ditu.live.com",
+            "*.t2.tiles.ditu.live.com",
+            "*.t3.tiles.ditu.live.com",
+            "*.tiles.ditu.live.com",
+            "3d.live.com",
+            "api.search.live.com",
+            "beta.search.live.com",
+            "cnweb.search.live.com",
+            "dev.live.com",
+            "ditu.live.com",
+            "farecast.live.com",
+            "image.live.com",
+            "images.live.com",
+            "local.live.com.au",
+            "localsearch.live.com",
+            "ls4d.search.live.com",
+            "mail.live.com",
+            "mapindia.live.com",
+            "local.live.com",
+            "maps.live.com",
+            "maps.live.com.au",
+            "mindia.live.com",
+            "news.live.com",
+            "origin.cnweb.search.live.com",
+            "preview.local.live.com",
+            "search.live.com",
+            "test.maps.live.com",
+            "video.live.com",
+            "videos.live.com",
+            "virtualearth.live.com",
+            "wap.live.com",
+            "webmaster.live.com",
+            "webmasters.live.com",
+            "www.local.live.com.au",
+            "www.maps.live.com.au"
+        ],
+        "file.x509.issuer.common_name": "Microsoft IT TLS CA 5",
+        "file.x509.issuer.country": "US",
+        "file.x509.issuer.locality": "Redmond",
+        "file.x509.issuer.organization": "Microsoft Corporation",
+        "file.x509.issuer.organizational_unit": "Microsoft IT",
+        "file.x509.issuer.state_or_province": "Washington",
+        "file.x509.not_after": "2019-07-10T17:47:08.000Z",
+        "file.x509.not_before": "2017-07-20T17:47:08.000Z",
+        "file.x509.public_key_algorithm": "rsaEncryption",
+        "file.x509.public_key_exponent": 65537,
+        "file.x509.public_key_size": 2048,
+        "file.x509.serial_number": "2D00003299D7071DB7D1708A42000000003299",
+        "file.x509.signature_algorithm": "SHA256-RSA",
+        "file.x509.subject.common_name": "www.bing.com",
+        "file.x509.version_number": "3",
         "fileset.name": "x509",
         "input.type": "log",
         "log.offset": 0,