[Filebeat] Add regenerated rsa2elk modules with subdomain processors (#…

…23035) Changed rsa2elk modules with registered_domain and subdomain every place we have those fields available in ECS.
elastic · Dec 14, 2020 · 2988c49 · 2988c49
1 parent 87ff5c0
commit 2988c49
Show file tree

Hide file tree

Showing 169 changed files with 35,146 additions and 18,534 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -752,6 +752,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add logic for external network.direction in sophos xg fileset {pull}22973[22973]
 - Add `http.request.mime_type` for Elasticsearch audit log fileset. {pull}22975[22975]
 - Add configuration option to set external and internal networks for panw panos fileset {pull}22998[22998]
+- Add `subbdomain` fields for rsa2elk modules. {pull}23035[23035]
 - Add subdomain enrichment for suricata/eve fileset. {pull}23011[23011]
 - Add subdomain enrichment for zeek/dns fileset. {pull}23011[23011]
 - Add `event.category` "configuration" to auditd module events. {pull}23010[23010]

diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json
@@ -551,4 +551,4 @@
         "trace.id": "Root=1-58337262-36d228ad5d99923122bbe354",
         "user_agent.original": "curl/7.46.0"
     }
-]
+]
diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json
@@ -584,4 +584,4 @@
         "trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3",
         "user_agent.original": "-"
     }
-]
+]
diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json
@@ -33,4 +33,4 @@
             "forwarded"
         ]
     }
-]
+]
diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json
@@ -28,4 +28,4 @@
             "forwarded"
         ]
     }
-]
+]
diff --git a/x-pack/filebeat/module/barracuda/spamfirewall/config/input.yml b/x-pack/filebeat/module/barracuda/spamfirewall/config/input.yml
@@ -39,6 +39,48 @@ processors:
 {{ if .community_id }}
 - community_id: ~
 {{ end }}
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: dns.question.name
+    target_field: dns.question.registered_domain
+    target_subdomain_field: dns.question.subdomain
+    target_etld_field: dns.question.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: client.domain
+    target_field: client.registered_domain
+    target_subdomain_field: client.subdomain
+    target_etld_field: client.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: server.domain
+    target_field: server.registered_domain
+    target_subdomain_field: server.subdomain
+    target_etld_field: server.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: destination.domain
+    target_field: destination.registered_domain
+    target_subdomain_field: destination.subdomain
+    target_etld_field: destination.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: source.domain
+    target_field: source.registered_domain
+    target_subdomain_field: source.subdomain
+    target_etld_field: source.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: url.domain
+    target_field: url.registered_domain
+    target_subdomain_field: url.subdomain
+    target_etld_field: url.top_level_domain
 - add_fields:
     target: ''
     fields:

diff --git a/x-pack/filebeat/module/barracuda/spamfirewall/config/liblogparser.js b/x-pack/filebeat/module/barracuda/spamfirewall/config/liblogparser.js
@@ -1012,15 +1012,15 @@ var ecs_mappings = {
     "ddomain": {to:[{field: "destination.domain", setter: fld_prio, prio: 0}]},
     "devicehostip": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 2},{field: "related.ip", setter: fld_append}]},
     "devicehostmac": {convert: to_mac, to:[{field: "host.mac", setter: fld_prio, prio: 0}]},
-    "dhost": {to:[{field: "destination.address", setter: fld_set}]},
+    "dhost": {to:[{field: "destination.address", setter: fld_set},{field: "related.hosts", setter: fld_append}]},
     "dinterface": {to:[{field: "observer.egress.interface.name", setter: fld_set}]},
     "direction": {to:[{field: "network.direction", setter: fld_set}]},
     "directory": {to:[{field: "file.directory", setter: fld_set}]},
     "dmacaddr": {convert: to_mac, to:[{field: "destination.mac", setter: fld_set}]},
     "dns.responsetype": {to:[{field: "dns.answers.type", setter: fld_set}]},
     "dns.resptext": {to:[{field: "dns.answers.name", setter: fld_set}]},
     "dns_querytype": {to:[{field: "dns.question.type", setter: fld_set}]},
-    "domain": {to:[{field: "server.domain", setter: fld_prio, prio: 0}]},
+    "domain": {to:[{field: "server.domain", setter: fld_prio, prio: 0},{field: "related.hosts", setter: fld_append}]},
     "domain.dst": {to:[{field: "destination.domain", setter: fld_prio, prio: 1}]},
     "domain.src": {to:[{field: "source.domain", setter: fld_prio, prio: 2}]},
     "domain_id": {to:[{field: "user.domain", setter: fld_set}]},
@@ -1030,6 +1030,7 @@ var ecs_mappings = {
     "dtransport": {convert: to_long, to:[{field: "destination.nat.port", setter: fld_prio, prio: 0}]},
     "ec_outcome": {to:[{field: "event.outcome", setter: fld_ecs_outcome}]},
     "event_description": {to:[{field: "message", setter: fld_prio, prio: 0}]},
+    "event_source": {to:[{field: "related.hosts", setter: fld_append}]},
     "event_time": {convert: to_date, to:[{field: "@timestamp", setter: fld_set}]},
     "event_type": {to:[{field: "event.action", setter: fld_prio, prio: 1}]},
     "extension": {to:[{field: "file.extension", setter: fld_prio, prio: 1}]},
@@ -1038,9 +1039,10 @@ var ecs_mappings = {
     "filename_size": {convert: to_long, to:[{field: "file.size", setter: fld_set}]},
     "filepath": {to:[{field: "file.path", setter: fld_set}]},
     "filetype": {to:[{field: "file.type", setter: fld_set}]},
+    "fqdn": {to:[{field: "related.hosts", setter: fld_append}]},
     "group": {to:[{field: "group.name", setter: fld_set}]},
     "groupid": {to:[{field: "group.id", setter: fld_set}]},
-    "host": {to:[{field: "host.name", setter: fld_prio, prio: 1}]},
+    "host": {to:[{field: "host.name", setter: fld_prio, prio: 1},{field: "related.hosts", setter: fld_append}]},
     "hostip": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 0},{field: "related.ip", setter: fld_append}]},
     "hostip_v6": {convert: to_ip, to:[{field: "host.ip", setter: fld_prio, prio: 1},{field: "related.ip", setter: fld_append}]},
     "hostname": {to:[{field: "host.name", setter: fld_prio, prio: 0}]},
@@ -1094,7 +1096,7 @@ var ecs_mappings = {
     "service.name": {to:[{field: "service.name", setter: fld_prio, prio: 0}]},
     "service_account": {to:[{field: "related.user", setter: fld_append},{field: "user.name", setter: fld_prio, prio: 7}]},
     "severity": {to:[{field: "log.level", setter: fld_set}]},
-    "shost": {to:[{field: "host.hostname", setter: fld_set},{field: "source.address", setter: fld_set}]},
+    "shost": {to:[{field: "host.hostname", setter: fld_set},{field: "source.address", setter: fld_set},{field: "related.hosts", setter: fld_append}]},
     "sinterface": {to:[{field: "observer.ingress.interface.name", setter: fld_set}]},
     "sld": {to:[{field: "url.registered_domain", setter: fld_set}]},
     "smacaddr": {convert: to_mac, to:[{field: "source.mac", setter: fld_set}]},
@@ -1119,9 +1121,10 @@ var ecs_mappings = {
     "user_id": {to:[{field: "user.id", setter: fld_prio, prio: 0}]},
     "username": {to:[{field: "related.user", setter: fld_append},{field: "user.name", setter: fld_prio, prio: 1}]},
     "version": {to:[{field: "observer.version", setter: fld_set}]},
-    "web_domain": {to:[{field: "url.domain", setter: fld_prio, prio: 1}]},
+    "web_domain": {to:[{field: "url.domain", setter: fld_prio, prio: 1},{field: "related.hosts", setter: fld_append}]},
     "web_extension": {to:[{field: "file.extension", setter: fld_prio, prio: 0}]},
     "web_query": {to:[{field: "url.query", setter: fld_prio, prio: 1}]},
+    "web_ref_domain": {to:[{field: "related.hosts", setter: fld_append}]},
     "web_referer": {to:[{field: "http.request.referrer", setter: fld_prio, prio: 0}]},
     "web_root": {to:[{field: "url.path", setter: fld_set}]},
     "webpage": {to:[{field: "file.name", setter: fld_prio, prio: 1}]},
@@ -2014,6 +2017,7 @@ function do_populate(evt, base, targets) {
         var mapping = targets[key];
         if (mapping === undefined) continue;
         var value = base[key];
+        if (value === "") continue;
         if (mapping.convert !== undefined) {
             value = mapping.convert(value);
             if (value === undefined) {

diff --git a/x-pack/filebeat/module/barracuda/spamfirewall/ingest/pipeline.yml b/x-pack/filebeat/module/barracuda/spamfirewall/ingest/pipeline.yml
@@ -55,14 +55,9 @@ processors:
         ignore_missing: true
   - append:
         field: related.hosts
-        value: '{{url.domain}}'
-        if: ctx?.url?.domain != null && ctx?.url?.domain != ""
-        allow_duplicates: false
-  - append:
-        field: related.hosts
-        value: '{{server.domain}}'
-        if: ctx?.server?.domain != null && ctx?.url?.domain != ""
+        value: '{{host.name}}'
         allow_duplicates: false
+        if: ctx.host?.name != null && ctx.host?.name != ''
 on_failure:
   - append:
         field: error.message

diff --git a/x-pack/filebeat/module/barracuda/spamfirewall/manifest.yml b/x-pack/filebeat/module/barracuda/spamfirewall/manifest.yml
@@ -7,7 +7,7 @@ var:
   - name: syslog_host
     default: localhost
   - name: syslog_port
-    default: 9524
+    default: 9540
   - name: input
     default: udp
   - name: community_id

diff --git a/x-pack/filebeat/module/barracuda/spamfirewall/test/generated.log-expected.json b/x-pack/filebeat/module/barracuda/spamfirewall/test/generated.log-expected.json
@@ -371,8 +371,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.action": " RECV",
@@ -873,8 +872,8 @@
         "observer.type": "Anti-Virus",
         "observer.vendor": "Barracuda",
         "related.hosts": [
-            "hitect",
-            "lit5929.test"
+            "lit5929.test",
+            "hitect"
         ],
         "related.ip": [
             "10.198.6.166"
@@ -896,6 +895,8 @@
         "rsa.time.endtime": "2017-02-03T21:16:50.000Z",
         "rsa.time.starttime": "2017-02-03T21:16:50.000Z",
         "server.domain": "lit5929.test",
+        "server.registered_domain": "lit5929.test",
+        "server.top_level_domain": "test",
         "service.type": "barracuda",
         "source.ip": [
             "10.198.6.166"
@@ -980,6 +981,9 @@
         "rsa.time.endtime": "2017-03-04T11:21:59.000Z",
         "rsa.time.starttime": "2017-03-04T11:21:59.000Z",
         "server.domain": "uptat3156.www5.test",
+        "server.registered_domain": "www5.test",
+        "server.subdomain": "uptat3156",
+        "server.top_level_domain": "test",
         "service.type": "barracuda",
         "source.ip": [
             "10.77.137.72"
@@ -1027,6 +1031,9 @@
         "rsa.time.endtime": "2017-03-18T18:24:33.000Z",
         "rsa.time.starttime": "2017-03-18T18:24:33.000Z",
         "server.domain": "neav6028.internal.domain",
+        "server.registered_domain": "internal.domain",
+        "server.subdomain": "neav6028",
+        "server.top_level_domain": "domain",
         "service.type": "barracuda",
         "source.ip": [
             "10.128.114.77"
@@ -1165,8 +1172,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.action": "deny",
@@ -1640,8 +1646,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.code": "web",
@@ -1844,8 +1849,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.action": " SCAN",
@@ -1861,8 +1865,8 @@
         "observer.type": "Anti-Virus",
         "observer.vendor": "Barracuda",
         "related.hosts": [
-            "aveni",
-            "oremagna3521.mail.home"
+            "oremagna3521.mail.home",
+            "aveni"
         ],
         "related.ip": [
             "10.29.155.171"
@@ -1884,6 +1888,9 @@
         "rsa.time.endtime": "2018-03-25T09:31:24.000Z",
         "rsa.time.starttime": "2018-03-25T09:31:24.000Z",
         "server.domain": "oremagna3521.mail.home",
+        "server.registered_domain": "mail.home",
+        "server.subdomain": "oremagna3521",
+        "server.top_level_domain": "home",
         "service.type": "barracuda",
         "source.ip": [
             "10.29.155.171"
@@ -1927,8 +1934,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.action": " RECV",
@@ -2044,8 +2050,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.code": "reports",
@@ -2720,8 +2725,7 @@
         "tags": [
             "barracuda.spamfirewall",
             "forwarded"
-        ],
-        "url.domain": ""
+        ]
     },
     {
         "event.action": "CHANGE",
@@ -3265,8 +3269,8 @@
         "observer.type": "Anti-Virus",
         "observer.vendor": "Barracuda",
         "related.hosts": [
-            "der",
-            "piciatis2460.api.host"
+            "piciatis2460.api.host",
+            "der"
         ],
         "related.ip": [
             "10.77.182.191"
@@ -3288,6 +3292,9 @@
         "rsa.time.endtime": "2019-11-30T00:21:57.000Z",
         "rsa.time.starttime": "2019-11-30T00:21:57.000Z",
         "server.domain": "piciatis2460.api.host",
+        "server.registered_domain": "api.host",
+        "server.subdomain": "piciatis2460",
+        "server.top_level_domain": "host",
         "service.type": "barracuda",
         "source.ip": [
             "10.77.182.191"

diff --git a/x-pack/filebeat/module/barracuda/waf/config/input.yml b/x-pack/filebeat/module/barracuda/waf/config/input.yml
@@ -39,6 +39,48 @@ processors:
 {{ if .community_id }}
 - community_id: ~
 {{ end }}
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: dns.question.name
+    target_field: dns.question.registered_domain
+    target_subdomain_field: dns.question.subdomain
+    target_etld_field: dns.question.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: client.domain
+    target_field: client.registered_domain
+    target_subdomain_field: client.subdomain
+    target_etld_field: client.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: server.domain
+    target_field: server.registered_domain
+    target_subdomain_field: server.subdomain
+    target_etld_field: server.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: destination.domain
+    target_field: destination.registered_domain
+    target_subdomain_field: destination.subdomain
+    target_etld_field: destination.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: source.domain
+    target_field: source.registered_domain
+    target_subdomain_field: source.subdomain
+    target_etld_field: source.top_level_domain
+- registered_domain:
+    ignore_missing: true
+    ignore_failure: true
+    field: url.domain
+    target_field: url.registered_domain
+    target_subdomain_field: url.subdomain
+    target_etld_field: url.top_level_domain
 - add_fields:
     target: ''
     fields:
-Original file line number
+Diff line change
@@ Expand Up / @@ -551,4 +551,4 @@ @@
             "trace.id": "Root=1-58337262-36d228ad5d99923122bbe354",
             "user_agent.original": "curl/7.46.0"
         }
-    ]
+    ]