From 4427f579e07912815b3b658156c8fdae6bac7aa7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 21 Jan 2019 21:50:22 +0100 Subject: [PATCH] Cherry-pick: #10030 and #9869 to 6.5: Handle IPv6 zone id in IIS filebeat ingest pipeline (#10057) IIS logs can include zone ids when using IPv6, this is correctly parsed but geoip processor doesn't accept these addresses. Create a temporary field without the zone id to be used by geoip processor. (cherry picked from commit d59ae8ce7ae21d84d49b92c0c9905fd1184b5c3b) (cherry picked from commit 5f1f6ca83d8d4f6786f1ef74931676eeea0f2b14) (cherry picked from commit e05f967b23be71cad20074e47ec6cf0e2de19854) Co-authored-by: Mathieu Martin --- CHANGELOG.next.asciidoc | 3 ++ .../module/iis/access/ingest/default.json | 16 +++++++- .../module/iis/access/test/test-ipv6zone.log | 5 +++ .../test/test-ipv6zone.log-expected.json | 39 +++++++++++++++++++ filebeat/module/iis/error/ingest/default.json | 16 +++++++- .../module/iis/error/test/ipv6_zone_id.log | 5 +++ .../error/test/ipv6_zone_id.log-expected.json | 16 ++++++++ 7 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 filebeat/module/iis/access/test/test-ipv6zone.log create mode 100644 filebeat/module/iis/access/test/test-ipv6zone.log-expected.json create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7a84773d07b..4d92e6d35fd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -40,11 +40,14 @@ https://github.com/elastic/beats/compare/v6.5.4...6.5[Check the HEAD diff] *Filebeat* +- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] error log: {pull}9869[9869] access log: {pull}10030[10030] + *Heartbeat* *Journalbeat* *Metricbeat* + - Fix panics in vsphere module when certain values where not returned by the API. {pull}9784[9784] *Packetbeat* diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json index 4cbe512f5c4..96f90b3be0c 100644 --- a/filebeat/module/iis/access/ingest/default.json +++ b/filebeat/module/iis/access/ingest/default.json @@ -40,10 +40,24 @@ "target_field": "iis.access.user_agent.original" } }, { - "geoip": { + "grok": { "field": "iis.access.remote_ip", + "patterns": [ + "%{NOZONEIP:iis.access.remote_ip_geoip}" + ], + "pattern_definitions": { + "NOZONEIP": "[^%]*" + } + } + }, { + "geoip": { + "field": "iis.access.remote_ip_geoip", "target_field": "iis.access.geoip" } + }, { + "remove": { + "field": "iis.access.remote_ip_geoip" + } }], "on_failure" : [{ "set" : { diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log b/filebeat/module/iis/access/test/test-ipv6zone.log new file mode 100644 index 00000000000..45519b2966f --- /dev/null +++ b/filebeat/module/iis/access/test/test-ipv6zone.log @@ -0,0 +1,5 @@ +#Software: Microsoft Internet Information Services 10.0 +#Version: 1.0 +#Date: 2018-01-01 10:11:12 +#Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken +2018-01-01 10:11:12 W3SVC1 MACHINE-NAME ::1%0 GET / - 80 - ::1%0 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789 diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json new file mode 100644 index 00000000000..c6c34189d27 --- /dev/null +++ b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json @@ -0,0 +1,39 @@ +[ + { + "@timestamp": "2018-01-01T10:11:12.000Z", + "fileset.module": "iis", + "fileset.name": "access", + "iis.access.body_received.bytes": "456", + "iis.access.body_sent.bytes": "123", + "iis.access.cookie": "-", + "iis.access.hostname": "example.com", + "iis.access.http_version": "1.1", + "iis.access.method": "GET", + "iis.access.port": "80", + "iis.access.query_string": "-", + "iis.access.referrer": "-", + "iis.access.remote_ip": "::1%0", + "iis.access.request_time_ms": "789", + "iis.access.response_code": "200", + "iis.access.server_ip": "::1%0", + "iis.access.server_name": "MACHINE-NAME", + "iis.access.site_name": "W3SVC1", + "iis.access.sub_status": "0", + "iis.access.url": "/", + "iis.access.user_agent.device": "Other", + "iis.access.user_agent.major": "70", + "iis.access.user_agent.minor": "0", + "iis.access.user_agent.name": "Chrome", + "iis.access.user_agent.original": "Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36", + "iis.access.user_agent.os": "Mac OS X 10.14.0", + "iis.access.user_agent.os_major": "10", + "iis.access.user_agent.os_minor": "14", + "iis.access.user_agent.os_name": "Mac OS X", + "iis.access.user_agent.patch": "3538", + "iis.access.user_name": "-", + "iis.access.win32_status": "0", + "input.type": "log", + "offset": 331, + "prospector.type": "log" + } +] \ No newline at end of file diff --git a/filebeat/module/iis/error/ingest/default.json b/filebeat/module/iis/error/ingest/default.json index 632e31d717f..af3c470afe7 100644 --- a/filebeat/module/iis/error/ingest/default.json +++ b/filebeat/module/iis/error/ingest/default.json @@ -28,10 +28,24 @@ "field": "iis.error.time" } }, { - "geoip": { + "grok": { "field": "iis.error.remote_ip", + "patterns": [ + "%{NOZONEIP:iis.error.remote_ip_geoip}" + ], + "pattern_definitions": { + "NOZONEIP": "[^%]*" + } + } + }, { + "geoip": { + "field": "iis.error.remote_ip_geoip", "target_field": "iis.error.geoip" } + }, { + "remove": { + "field": "iis.error.remote_ip_geoip" + } }], "on_failure" : [{ "set" : { diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log b/filebeat/module/iis/error/test/ipv6_zone_id.log new file mode 100644 index 00000000000..436e133e344 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log @@ -0,0 +1,5 @@ +#Software: Microsoft HTTP API 2.0 +#Version: 1.0 +#Date: 2018-12-30 13:48:36 +#Fields: date time c-ip c-port s-ip s-port cs-version cs-method cs-uri streamid sc-status s-siteid s-reason s-queuename +2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle - diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json new file mode 100644 index 00000000000..13c8b85e039 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -0,0 +1,16 @@ +[ + { + "@timestamp": "2018-12-30T14:22:07.000Z", + "fileset.module": "iis", + "fileset.name": "error", + "iis.error.queue_name": "-", + "iis.error.reason_phrase": "Timer_ConnectionIdle", + "iis.error.remote_ip": "::1%0", + "iis.error.remote_port": "49958", + "iis.error.server_ip": "::1%0", + "iis.error.server_port": "80", + "input.type": "log", + "offset": 195, + "prospector.type": "log" + } +] \ No newline at end of file