[Filebeat] Update Fortinet Ingest Pipeline (elastic#24816) (elastic#2…

…6576)

* 22136: Update Fortinet Ingest Pipeline

* Update Pipelines

* Additional updates

* Set virus/ips subtypes to event.kind: alert

* update fields

* Consolidate processors to script

* Update event.outcome logic

* replace hashmap

* update event.outcome

* cleanup

* Added Changes for elastic#25254

* regenerate data

* update changelog

* remove extra items in changelog

(cherry picked from commit 890e473)

Co-authored-by: Alex Resnick <adr8292@gmail.com>

CHANGELOG.next.asciidoc

@@ -76,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299]
 - All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699]
 - Deprecated the cyberark module (replaced by cyberarkpas). {issue}25261[25261] {pull}25505[25505]
+- Change source field for `event.action` in `fortinet.firewall` module to `fortinet.firewall.action` instead of `fortinet.firewall.eventtype`. {pull}24816[24816]
 
 *Heartbeat*
 - Add support for screenshot blocks and use newer synthetics flags that only works in newer synthetics betas. {pull}25808[25808]
@@ -603,6 +604,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
 - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
+- Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -62763,6 +62763,16 @@ type: keyword
 ESP Transform
 
 
+type: keyword
+
+--
+
+*`fortinet.firewall.eventtype`*::
++
+--
+UTM Event Type
+
+
 type: keyword
 
 --
@@ -65243,6 +65253,16 @@ type: integer
 Security action performed by UTM
 
 
+type: keyword
+
+--
+
+*`fortinet.firewall.utmref`*::
++
+--
+Reference to UTM
+
+
 type: keyword
 
 --

x-pack/filebeat/module/fortinet/firewall/_meta/fields.yml

@@ -766,6 +766,11 @@
           description: >
             ESP Transform
 
+        - name: eventtype
+          type: keyword
+          description: >
+            UTM Event Type
+
         - name: exch
           type: keyword
           description: >
@@ -2006,6 +2011,11 @@
           description: >
             Security action performed by UTM
 
+        - name: utmref
+          type: keyword
+          description: >
+            Reference to UTM
+
         - name: vap
           type: keyword
           description: >

x-pack/filebeat/module/fortinet/firewall/ingest/event.yml

@@ -3,6 +3,10 @@ processors:
 - set:
     field: event.kind
     value: event
+- set:
+    field: event.action
+    value: "{{fortinet.firewall.action}}"
+    ignore_empty_value: true
 - set:
     field: event.outcome
     value: failure
@@ -16,24 +20,29 @@ processors:
     value:
     - user
     - start
+    allow_duplicates: false
     if: "['FSSO-logon', 'auth-logon'].contains(ctx.fortinet?.firewall?.action)"
 - append:
     field: event.type
     value:
     - user
     - end
+    allow_duplicates: false
     if: "['FSSO-logoff', 'auth-logout'].contains(ctx.fortinet?.firewall?.action)"
 - append:
     field: event.type
     value: connection
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'vpn'"
 - append:
     field: event.category
     value: network
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'vpn'"
 - append:
     field: event.type
     value: info
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.action == 'perf-stats'"
 - append:
     field: event.category
@@ -42,16 +51,19 @@ processors:
 - append:
     field: event.type
     value: info
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'update'"
 - append:
     field: event.category
     value:
     - host
     - malware
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'update'"
 - append:
     field: event.category
     value: authentication
+    allow_duplicates: false
     if: "ctx.fortinet?.firewall?.subtype == 'user'"
 - rename:
     field: fortinet.firewall.dstip
@@ -95,10 +107,6 @@ processors:
     target_field: destination.domain
     ignore_missing: true
     if: "ctx.destination?.address == null"
-- rename:
-    field: fortinet.firewall.group
-    target_field: source.user.group.name
-    ignore_missing: true
 - convert:
     field: fortinet.firewall.sentbyte
     target_field: source.bytes
@@ -144,10 +152,6 @@ processors:
     field: fortinet.firewall.saddr
     target_field: source.address
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.agent
-    target_field: user_agent.original
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.file
     target_field: file.name
@@ -167,18 +171,10 @@ processors:
     target_field: event.code
     ignore_missing: true
     if: "ctx.event?.code == null"
-- rename:
-    field: fortinet.firewall.msg
-    target_field: message
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.policyid
     target_field: rule.id
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.proto
-    target_field: network.iana_number
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.dir
     target_field: network.direction
@@ -207,21 +203,10 @@ processors:
         return
       }
       ctx.network.direction = k;
-- rename:
-    field: fortinet.firewall.service
-    target_field: network.protocol
-    ignore_missing: true
-- lowercase:
-    field: network.protocol
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.error_num
     target_field: error.code
     ignore_missing: true
-- rename:
-    field: fortinet.firewall.hostname
-    target_field: url.domain
-    ignore_missing: true
 - rename:
     field: fortinet.firewall.logdesc
     target_field: rule.description

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -24,7 +24,7 @@ processors:
     source: |
       def fw = ctx?.fortinet?.firewall;
       if (fw != null) {
-        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A");
+        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A" || entry.getValue() == "undefined");
       }
 - set:
     field: observer.vendor
@@ -35,12 +35,6 @@ processors:
 - set:
     field: observer.type
     value: firewall
-- set:
-    field: event.module
-    value: fortinet
-- set:
-    field: event.dataset
-    value: fortinet.firewall
 - set:
     field: event.timezone
     value: "{{fortinet.firewall.tz}}"
@@ -198,18 +192,7 @@ processors:
       )
 - remove:
     field:
-      - _temp.time
-      - _temp
       - message
-      - syslog5424_sd
-      - syslog5424_pri
-      - fortinet.firewall.tz
-      - fortinet.firewall.date
-      - fortinet.firewall.devid
-      - fortinet.firewall.eventtime
-      - fortinet.firewall.time
-      - fortinet.firewall.duration
-      - host
     ignore_missing: true
 - pipeline:
     name: '{< IngestPipeline "event" >}'
@@ -220,6 +203,73 @@ processors:
 - pipeline:
     name: '{< IngestPipeline "utm" >}'
     if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
+- rename:
+    field: fortinet.firewall.reason
+    target_field: event.reason
+    ignore_missing: true
+- rename:
+    field: fortinet.firewall.msg
+    target_field: message
+    ignore_missing: true
+- rename:
+    field: fortinet.firewall.proto
+    target_field: network.iana_number
+    ignore_missing: true
+- script:
+    lang: painless
+    ignore_failure: true
+    if: ctx?.network?.iana_number != null
+    source: |
+      if (ctx?.network == null) {
+          ctx.network = new HashMap();
+      }
+      def iana_number = ctx.network.iana_number;
+      if (iana_number == '1') {
+        ctx.network.transport = 'icmp';
+      } else if (iana_number == '2') {
+        ctx.network.transport = 'igmp';
+      } else if (iana_number == '6') {
+        ctx.network.transport = 'tcp';
+      } else if (iana_number == '17') {
+        ctx.network.transport = 'udp';
+      } else if (iana_number == '58') {
+        ctx.network.transport = 'ipv6-icmp';
+      }
+- rename:
+    field: fortinet.firewall.group
+    target_field: source.user.group.name
+    ignore_missing: true
+- uri_parts:
+    field: fortinet.firewall.url
+    remove_if_successful: true
+    ignore_failure: true
+    if: "ctx.fortinet?.firewall?.url != null"
+- set:
+    field: url.domain
+    value: "{{fortinet.firewall.hostname}}"
+    ignore_empty_value: true
+    if: "ctx?.url?.domain == null"
+- rename:
+    field: fortinet.firewall.service
+    target_field: network.protocol
+    ignore_missing: true
+- lowercase:
+    field: network.protocol
+    ignore_missing: true
+- set:
+    field: network.type
+    value: ipv4
+    if: (ctx.source?.ip != null && ctx.source?.ip.contains('.')) || (ctx.destination?.ip != null && ctx.destination?.ip.contains('.'))
+- set:
+    field: network.type
+    value: ipv6
+    if: ctx.source?.ip != null && ctx.source?.ip.contains(':') || (ctx.destination?.ip != null && ctx.destination?.ip.contains(':'))
+- community_id:
+    ignore_missing: true
+    ignore_failure: true
+- user_agent:
+    field: fortinet.firewall.agent
+    ignore_missing: true
 - convert:
     field: fortinet.firewall.quotamax
     type: long
@@ -315,30 +365,37 @@ processors:
 - append:
     field: related.ip
     value: "{{source.ip}}"
+    allow_duplicates: false
     if: "ctx.source?.ip != null"
 - append:
     field: related.ip
     value: "{{destination.ip}}"
+    allow_duplicates: false
     if: "ctx.destination?.ip != null"
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 - append:
     field: related.user
     value: "{{destination.user.name}}"
+    allow_duplicates: false
     if: "ctx.destination?.user?.name != null"
 - append:
     field: related.hosts
     value: "{{destination.address}}"
+    allow_duplicates: false
     if: "ctx.destination?.address != null"
 - append:
     field: related.hosts
     value: "{{source.address}}"
+    allow_duplicates: false
     if: "ctx.source?.address != null"
 - append:
     field: related.hosts
     value: "{{dns.question.name}}"
+    allow_duplicates: false
     if: "ctx.dns?.question?.name != null"
 - script:
     lang: painless
@@ -354,6 +411,45 @@ processors:
           }
         }
       }
+- remove:
+    field:
+      - _temp
+      - syslog5424_sd
+      - syslog5424_pri
+      - fortinet.firewall.tz
+      - fortinet.firewall.date
+      - fortinet.firewall.devid
+      - fortinet.firewall.eventtime
+      - fortinet.firewall.time
+      - fortinet.firewall.duration
+      - host
+      - fortinet.firewall.hostname
+      - fortinet.firewall.agent
+    ignore_missing: true
+- script:
+    lang: painless
+    description: This script processor iterates over the whole document to remove fields with null values.
+    source: |
+      void handleMap(Map map) {
+        for (def x : map.values()) {
+          if (x instanceof Map) {
+              handleMap(x);
+          } else if (x instanceof List) {
+              handleList(x);
+          }
+        }
+        map.values().removeIf(v -> v == null);
+      }
+      void handleList(List list) {
+        for (def x : list) {
+            if (x instanceof Map) {
+                handleMap(x);
+            } else if (x instanceof List) {
+                handleList(x);
+            }
+        }
+      }
+      handleMap(ctx);
 on_failure:
 - set:
     field: error.message