From d59ae8ce7ae21d84d49b92c0c9905fd1184b5c3b Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor <jaime.soriano@elastic.co> Date: Mon, 7 Jan 2019 20:00:14 +0100 Subject: [PATCH] Handle IPv6 zone id in IIS filebeat ingest pipeline (#9869) IIS logs can include zone ids when using IPv6, this is correctly parsed but geoip processor doesn't accept these addresses. Create a temporary field without the zone id to be used by geoip processor. --- CHANGELOG.next.asciidoc | 1 + filebeat/module/iis/error/ingest/default.json | 16 +++++++++++++++- filebeat/module/iis/error/test/ipv6_zone_id.log | 5 +++++ .../error/test/ipv6_zone_id.log-expected.json | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log create mode 100644 filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3adfab581ae..01dfe76d85c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] +- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869] *Heartbeat* diff --git a/filebeat/module/iis/error/ingest/default.json b/filebeat/module/iis/error/ingest/default.json index 632e31d717f..af3c470afe7 100644 --- a/filebeat/module/iis/error/ingest/default.json +++ b/filebeat/module/iis/error/ingest/default.json @@ -28,10 +28,24 @@ "field": "iis.error.time" } }, { - "geoip": { + "grok": { "field": "iis.error.remote_ip", + "patterns": [ + "%{NOZONEIP:iis.error.remote_ip_geoip}" + ], + "pattern_definitions": { + "NOZONEIP": "[^%]*" + } + } + }, { + "geoip": { + "field": "iis.error.remote_ip_geoip", "target_field": "iis.error.geoip" } + }, { + "remove": { + "field": "iis.error.remote_ip_geoip" + } }], "on_failure" : [{ "set" : { diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log b/filebeat/module/iis/error/test/ipv6_zone_id.log new file mode 100644 index 00000000000..436e133e344 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log @@ -0,0 +1,5 @@ +#Software: Microsoft HTTP API 2.0 +#Version: 1.0 +#Date: 2018-12-30 13:48:36 +#Fields: date time c-ip c-port s-ip s-port cs-version cs-method cs-uri streamid sc-status s-siteid s-reason s-queuename +2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle - diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json new file mode 100644 index 00000000000..99c1a3bd093 --- /dev/null +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -0,0 +1,16 @@ +[ + { + "@timestamp": "2018-12-30T14:22:07.000Z", + "ecs.version": "1.0.0-beta2", + "event.dataset": "error", + "event.module": "iis", + "iis.error.queue_name": "-", + "iis.error.reason_phrase": "Timer_ConnectionIdle", + "iis.error.remote_ip": "::1%0", + "iis.error.remote_port": "49958", + "iis.error.server_ip": "::1%0", + "iis.error.server_port": "80", + "input.type": "log", + "log.offset": 195 + } +] \ No newline at end of file