From ca182dab34bb186cf0ca846539c94914e02a5df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Apr 2018 13:07:23 +0200 Subject: [PATCH 1/6] support `log_format combined` setting of nginx access logs --- filebeat/module/nginx/access/ingest/default.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index ecb3df51b92..f98da37d989 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -4,13 +4,25 @@ "grok": { "field": "message", "patterns":[ - "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" + "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" ], "pattern_definitions": { "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" }, "ignore_missing": true } + }, { + "grok": { + "field": "nginx.access.info", + "patterns": [ + "(%{WORD:nginx.access.method} )??%{DATA:nginx.access.url}( HTTP/%{NUMBER:nginx.access.http_version})?" + ], + "ignore_missing": true + } + }, { + "remove": { + "field": "nginx.access.info" + } }, { "split": { "field": "nginx.access.remote_ip_list", From 48d620a712278b8a96555660a262557f2196072e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Apr 2018 16:57:14 +0200 Subject: [PATCH 2/6] add test case --- filebeat/module/nginx/access/test/test.log | 1 + .../nginx/access/test/test.log-expected.json | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log index e303a6d516d..d107d245269 100644 --- a/filebeat/module/nginx/access/test/test.log +++ b/filebeat/module/nginx/access/test/test.log @@ -4,3 +4,4 @@ 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" +127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-" diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 62efe0bb908..8c846064a2a 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -362,5 +362,53 @@ "name" : "access" } } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtqbaTipsE", + "_score" : 1.0, + "_source": { + "@timestamp": "2018-04-12T07:48:40.000Z", + "nginx": { + "access": { + "body_sent": { + "bytes": "0" + }, + "referrer": "-", + "remote_ip": "127.0.0.1", + "remote_ip_list": [ + "127.0.0.1" + ], + "response_code": "400", + "url": "", + "user_agent": { + "device": "Other", + "name": "Other", + "os": "Other", + "os_name": "Other" + }, + "user_name": "-" + } + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "input" : { + "type" : "log" + }, + "read_timestamp": "2018-04-13T11:13:43.103Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } } ] From 19e51d97c1b1194bcbec252c74972d492de9e050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Apr 2018 16:57:23 +0200 Subject: [PATCH 3/6] add to changelog --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 221823cc3fe..ccb1cd4cb2d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -122,6 +122,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Make registry file permission configurable. {pull}6455[6455] - Add MongoDB module. {pull}6283[6238] - Add Ingest pipeline loading to setup. {pull}6814[6814] +- Add support of log_format combined to NGINX access logs. {pull}6858[6858] *Heartbeat* From 752a2f0f9349a9b4f01e8a39e27bfeabc6698b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Apr 2018 17:16:58 +0200 Subject: [PATCH 4/6] fix pattern --- filebeat/module/nginx/access/ingest/default.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index f98da37d989..fd961356529 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -15,9 +15,9 @@ "grok": { "field": "nginx.access.info", "patterns": [ - "(%{WORD:nginx.access.method} )??%{DATA:nginx.access.url}( HTTP/%{NUMBER:nginx.access.http_version})?" + "(?:%{WORD:nginx.access.method} )(?:%{DATA:nginx.access.url} )(?:HTTP/%{NUMBER:nginx.access.http_version})" ], - "ignore_missing": true + "ignore_missing": false } }, { "remove": { From 07f941fbb1265a38ad73fcbbe3409505ab04d496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Fri, 13 Apr 2018 19:10:10 +0200 Subject: [PATCH 5/6] really fix pattern --- filebeat/module/nginx/access/ingest/default.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index fd961356529..1412b4f5102 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -15,9 +15,10 @@ "grok": { "field": "nginx.access.info", "patterns": [ + "", "(?:%{WORD:nginx.access.method} )(?:%{DATA:nginx.access.url} )(?:HTTP/%{NUMBER:nginx.access.http_version})" ], - "ignore_missing": false + "ignore_missing": true } }, { "remove": { From 670ed57440537c2eaf59bc10df2cd035dd9dcb7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 16 Apr 2018 12:20:22 +0200 Subject: [PATCH 6/6] fix expectes --- .../module/nginx/access/ingest/default.json | 4 +- .../nginx/access/test/test.log-expected.json | 42 +++++++++---------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index 1412b4f5102..04ae1197e69 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -15,8 +15,8 @@ "grok": { "field": "nginx.access.info", "patterns": [ - "", - "(?:%{WORD:nginx.access.method} )(?:%{DATA:nginx.access.url} )(?:HTTP/%{NUMBER:nginx.access.http_version})" + "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}", + "" ], "ignore_missing": true } diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 8c846064a2a..36f5949265c 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -369,28 +369,26 @@ "_id" : "AVxWUuZ8OMOtqbaTipsE", "_score" : 1.0, "_source": { - "@timestamp": "2018-04-12T07:48:40.000Z", - "nginx": { - "access": { - "body_sent": { - "bytes": "0" - }, - "referrer": "-", - "remote_ip": "127.0.0.1", - "remote_ip_list": [ - "127.0.0.1" - ], - "response_code": "400", - "url": "", - "user_agent": { - "device": "Other", - "name": "Other", - "os": "Other", - "os_name": "Other" - }, - "user_name": "-" - } - } + "@timestamp": "2018-04-12T07:48:40.000Z", + "nginx": { + "access": { + "body_sent": { + "bytes": "0" + }, + "referrer": "-", + "remote_ip": "127.0.0.1", + "remote_ip_list": [ + "127.0.0.1" + ], + "response_code": "400", + "user_agent": { + "device": "Other", + "name": "Other", + "os": "Other", + "os_name": "Other" + }, + "user_name": "-" + } }, "beat" : { "hostname" : "a-mac-with-esc-key-2.local",