From 1f68e2b0f8968f07b33147ef80d4d050720dadf6 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 15 Mar 2019 13:21:35 -0600 Subject: [PATCH] Change URLPATH grok pattern to support brackets (#11252) * Change URLPATH grok pattern to support [] * Update URIPATHWITHBRACKET with the current URIPATH * Add changelog --- CHANGELOG.next.asciidoc | 1 + .../module/iis/access/ingest/default.json | 7 ++- filebeat/module/iis/access/test/test.log | 2 + .../iis/access/test/test.log-expected.json | 58 +++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bd7a850a489f..90dcb7e94fa0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix a bug when converting NetFlow fields to snake_case. {pull}10950[10950] - Add on_failure handler for Zeek ingest pipelines. Fix one field name error for notice and add an additional test case. {issue}11004[11004] {pull}11105[11105] +- Change URLPATH grok pattern to support brackets. {issue}11135[11135] {pull}11252[11252] *Heartbeat* diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json index 32170189a752..fc4ad7538281 100644 --- a/filebeat/module/iis/access/ingest/default.json +++ b/filebeat/module/iis/access/ingest/default.json @@ -4,11 +4,14 @@ "grok": { "field": "message", "patterns":[ - "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATHWITHBRACKET:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} HTTP/%{NUMBER:http.version} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:destination.address}\\]\\(http://%{IPORHOST:destination.address}\\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \\[%{IPORHOST:source.address}\\]\\(http://%{IPORHOST:source.address}\\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}" - ], + ], + "pattern_definitions": { + "URIPATHWITHBRACKET": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-\\[\\]]*)+" + }, "ignore_missing": true } }, { diff --git a/filebeat/module/iis/access/test/test.log b/filebeat/module/iis/access/test/test.log index d5f30c9f0c23..c054eaf65578 100644 --- a/filebeat/module/iis/access/test/test.log +++ b/filebeat/module/iis/access/test/test.log @@ -13,3 +13,5 @@ #Date: 2018-01-01 10:11:12 #Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken 2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789 +2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0 +2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0 diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 9bc8586de201..0efae3c6dde5 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -111,5 +111,63 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.14.0", "user_agent.version": "70.0.3538" + }, + { + "@timestamp": "2018-12-31T12:52:33.000Z", + "destination.address": "10.44.0.136", + "destination.ip": "10.44.0.136", + "destination.port": 443, + "ecs.version": "1.0.0", + "event.dataset": "iis.access", + "event.duration": 0, + "event.module": "iis", + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 401, + "iis.access.sub_status": 0, + "iis.access.win32_status": 0, + "input.type": "log", + "log.offset": 1447, + "service.type": "iis", + "source.address": "10.50.6.188", + "source.ip": "10.50.6.188", + "url.path": "/", + "url.query": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "IE", + "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", + "user_agent.os.name": "Windows XP", + "user_agent.version": "8.0" + }, + { + "@timestamp": "2018-12-31T12:52:33.000Z", + "destination.address": "10.44.0.136", + "destination.ip": "10.44.0.136", + "destination.port": 443, + "ecs.version": "1.0.0", + "event.dataset": "iis.access", + "event.duration": 0, + "event.module": "iis", + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 404, + "iis.access.sub_status": 0, + "iis.access.win32_status": 2, + "input.type": "log", + "log.offset": 1802, + "service.type": "iis", + "source.address": "10.50.6.188", + "source.ip": "10.50.6.188", + "url.path": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", + "url.query": "-", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "IE", + "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", + "user_agent.os.name": "Windows XP", + "user_agent.version": "8.0" } ] \ No newline at end of file