[filebeat][fortinet] Use default add_locale for fortinet.firewall (el…

…astic#26524) * Use default add_locale for fortinet.firewall * Use event.timezone in all date processors
marc-gr · Jun 30, 2021 · 5d258c8 · 5d258c8
1 parent 26325b0
commit 5d258c8
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 73 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -842,6 +842,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
 - Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
+- Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/modules/fortinet.asciidoc b/filebeat/docs/modules/fortinet.asciidoc
@@ -27,7 +27,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-This module has been tested against FortiOS version 6.0.x and 6.2.x. 
+This module has been tested against FortiOS version 6.0.x and 6.2.x.
 Versions above this are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]
@@ -51,6 +51,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/timezone-support.asciidoc[]
+
 *`var.input`*::
 
 The input to use, can be either the value `tcp`, `udp` or `file`.

diff --git a/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc b/x-pack/filebeat/module/fortinet/_meta/docs.asciidoc
@@ -22,7 +22,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-This module has been tested against FortiOS version 6.0.x and 6.2.x. 
+This module has been tested against FortiOS version 6.0.x and 6.2.x.
 Versions above this are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]
@@ -46,6 +46,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/timezone-support.asciidoc[]
+
 *`var.input`*::
 
 The input to use, can be either the value `tcp`, `udp` or `file`.

diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml
@@ -25,6 +25,7 @@ tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 
 processors:
+  - add_locale: ~
   - add_fields:
       target: ''
       fields:

diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml
@@ -41,12 +41,12 @@ processors:
     ignore_empty_value: true
 - set:
     field: _temp.time
-    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null"
+    value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}} {{event.timezone}}"
+    if: "ctx.event?.timezone != null"
 - set:
     field: _temp.time
     value: "{{fortinet.firewall.date}} {{fortinet.firewall.time}}"
-    if: "ctx.fortinet?.firewall?.tz == null"
+    if: "ctx.event?.timezone == null"
 - date:
     field: _temp.time
     target_field: "@timestamp"
@@ -55,8 +55,8 @@ processors:
     - yyyy-MM-dd HH:mm:ss Z
     - yyyy-MM-dd HH:mm:ss z
     - ISO8601
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx.fortinet?.firewall?.tz != null"
+    timezone: "{{event.timezone}}"
+    if: "ctx.event?.timezone != null"
 - date:
     field: _temp.time
     target_field: "@timestamp"
@@ -65,7 +65,7 @@ processors:
     - yyyy-MM-dd HH:mm:ss Z
     - yyyy-MM-dd HH:mm:ss z
     - ISO8601
-    if: "ctx.fortinet?.firewall?.tz == null"
+    if: "ctx.event?.timezone == null"
 - gsub:
     field: fortinet.firewall.eventtime
     pattern: "\\d{6}$"
@@ -76,27 +76,27 @@ processors:
     target_field: event.start
     formats:
     - UNIX_MS
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    timezone: "{{event.timezone}}"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    timezone: "{{fortinet.firewall.tz}}"
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    timezone: "{{event.timezone}}"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX_MS
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() > 11"
 - date:
     field: fortinet.firewall.eventtime
     target_field: event.start
     formats:
     - UNIX
-    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
+    if: "ctx?.fortinet?.firewall?.eventtime != null && ctx.event?.timezone == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11"
 - script:
     lang: painless
     source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000"

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json
@@ -635,7 +635,7 @@
         ]
     },
     {
-        "@timestamp": "2020-11-02T08:11:38.000Z",
+        "@timestamp": "2020-11-02T08:11:38.000-02:00",
         "destination.as.number": 15169,
         "destination.as.organization.name": "Google LLC",
         "destination.geo.continent_name": "North America",
@@ -654,6 +654,7 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection"
         ],
@@ -696,15 +697,16 @@
         ]
     },
     {
-        "@timestamp": "2019-05-13T11:20:54.000Z",
+        "@timestamp": "2019-05-13T11:20:54.000-02:00",
         "destination.ip": "172.16.200.2",
         "event.action": "login",
         "event.code": "0100032001",
         "event.dataset": "fortinet.firewall",
         "event.kind": "event",
         "event.module": "fortinet",
         "event.reason": "none",
-        "event.start": "2019-05-13T18:20:54.587Z",
+        "event.start": "2019-05-13T16:20:54.587-02:00",
+        "event.timezone": "-02:00",
         "fileset.name": "firewall",
         "fortinet.firewall.action": "login",
         "fortinet.firewall.method": "ssh",
@@ -740,7 +742,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-13T14:21:42.000Z",
+        "@timestamp": "2019-05-13T14:21:42.000-02:00",
         "destination.as.number": 7065,
         "destination.as.organization.name": "Sonoma Interconnect",
         "destination.geo.city_name": "North Highlands",
@@ -762,7 +764,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-13T21:21:42.722Z",
+        "event.start": "2019-05-13T19:21:42.722-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection"
         ],
@@ -813,7 +816,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-13T15:55:56.000Z",
+        "@timestamp": "2019-05-13T15:55:56.000-02:00",
         "destination.ip": "172.16.200.55",
         "event.action": "authentication",
         "event.category": [
@@ -823,7 +826,8 @@
         "event.dataset": "fortinet.firewall",
         "event.kind": "event",
         "event.module": "fortinet",
-        "event.start": "2019-05-13T22:55:56.913Z",
+        "event.start": "2019-05-13T20:55:56.913-02:00",
+        "event.timezone": "-02:00",
         "fileset.name": "firewall",
         "fortinet.firewall.action": "authentication",
         "fortinet.firewall.authproto": "TELNET(10.1.100.11)",
@@ -859,13 +863,14 @@
         ]
     },
     {
-        "@timestamp": "2019-05-14T08:32:13.000Z",
+        "@timestamp": "2019-05-14T08:32:13.000-02:00",
         "event.action": "add",
         "event.code": "0107045057",
         "event.dataset": "fortinet.firewall",
         "event.kind": "event",
         "event.module": "fortinet",
-        "event.start": "2019-05-14T15:32:13.900Z",
+        "event.start": "2019-05-14T13:32:13.900-02:00",
+        "event.timezone": "-02:00",
         "fileset.name": "firewall",
         "fortinet.firewall.action": "add",
         "fortinet.firewall.connection_type": "sslvpn",
@@ -898,13 +903,14 @@
         ]
     },
     {
-        "@timestamp": "2019-05-14T08:19:38.000Z",
+        "@timestamp": "2019-05-14T08:19:38.000-02:00",
         "event.action": "close",
         "event.code": "0107045058",
         "event.dataset": "fortinet.firewall",
         "event.kind": "event",
         "event.module": "fortinet",
-        "event.start": "2019-05-14T15:19:39.037Z",
+        "event.start": "2019-05-14T13:19:39.037-02:00",
+        "event.timezone": "-02:00",
         "fileset.name": "firewall",
         "fortinet.firewall.action": "close",
         "fortinet.firewall.connection_type": "sslvpn",

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2020-04-23T01:16:08.000Z",
+        "@timestamp": "2020-04-23T01:16:08.000-02:00",
         "destination.as.number": 15169,
         "destination.as.organization.name": "Google LLC",
         "destination.bytes": 0,
@@ -21,7 +21,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2020-06-24T01:16:08.000Z",
+        "event.start": "2020-06-23T23:16:08.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "denied",
@@ -581,7 +582,7 @@
         ]
     },
     {
-        "@timestamp": "2019-03-31T06:42:54.000Z",
+        "@timestamp": "2019-03-31T06:42:54.000-02:00",
         "destination.bytes": 0,
         "destination.ip": "230.1.1.2",
         "destination.packets": 0,
@@ -596,7 +597,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-03-31T13:42:52.000Z",
+        "event.start": "2019-03-31T11:42:52.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "allowed",
             "connection",
@@ -644,7 +646,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-10T14:18:54.000Z",
+        "@timestamp": "2019-05-10T14:18:54.000-02:00",
         "destination.as.number": 20940,
         "destination.as.organization.name": "Akamai International B.V.",
         "destination.bytes": 0,
@@ -666,7 +668,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-10T21:18:54.021Z",
+        "event.start": "2019-05-10T19:18:54.021-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "end"
@@ -732,7 +735,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-13T11:45:04.000Z",
+        "@timestamp": "2019-05-13T11:45:04.000-02:00",
         "destination.bytes": 2286,
         "destination.ip": "172.16.200.55",
         "destination.packets": 6,
@@ -747,7 +750,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-13T18:45:04.815Z",
+        "event.start": "2019-05-13T16:45:04.815-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "denied",
@@ -819,7 +823,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-13T16:29:50.000Z",
+        "@timestamp": "2019-05-13T16:29:50.000-02:00",
         "destination.as.number": 42831,
         "destination.as.organization.name": "UK Dedicated Servers Limited",
         "destination.bytes": 3138,
@@ -844,7 +848,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-13T23:29:50.452Z",
+        "event.start": "2019-05-13T21:29:50.452-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "denied",
@@ -906,7 +911,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-15T17:58:10.000Z",
+        "@timestamp": "2019-05-15T17:58:10.000-02:00",
         "destination.bytes": 9112,
         "destination.ip": "172.16.200.55",
         "destination.packets": 8,
@@ -921,7 +926,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-16T00:58:09.000Z",
+        "event.start": "2019-05-15T22:58:09.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "denied",
@@ -984,7 +990,7 @@
         ]
     },
     {
-        "@timestamp": "2019-05-15T17:45:34.000Z",
+        "@timestamp": "2019-05-15T17:45:34.000-02:00",
         "destination.as.number": 16509,
         "destination.as.organization.name": "Amazon.com, Inc.",
         "destination.bytes": 5266,
@@ -1009,7 +1015,8 @@
         "event.kind": "event",
         "event.module": "fortinet",
         "event.outcome": "success",
-        "event.start": "2019-05-16T00:45:34.000Z",
+        "event.start": "2019-05-15T22:45:34.000-02:00",
+        "event.timezone": "-02:00",
         "event.type": [
             "connection",
             "denied",