From 13fca49e86a3bd8c80cc4f76bf74088bac139531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 16 Apr 2018 16:55:15 +0200 Subject: [PATCH] Support `log_format combined` setting of NGINX access logs (#6858) Support `log_format combined` setting of nginx access logs --- CHANGELOG.asciidoc | 1 + .../module/nginx/access/ingest/default.json | 15 +++++- filebeat/module/nginx/access/test/test.log | 1 + .../nginx/access/test/test.log-expected.json | 46 +++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 221823cc3fe4..ccb1cd4cb2d9 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -122,6 +122,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Make registry file permission configurable. {pull}6455[6455] - Add MongoDB module. {pull}6283[6238] - Add Ingest pipeline loading to setup. {pull}6814[6814] +- Add support of log_format combined to NGINX access logs. {pull}6858[6858] *Heartbeat* diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index ecb3df51b92b..04ae1197e696 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -4,13 +4,26 @@ "grok": { "field": "message", "patterns":[ - "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" + "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" ], "pattern_definitions": { "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" }, "ignore_missing": true } + }, { + "grok": { + "field": "nginx.access.info", + "patterns": [ + "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}", + "" + ], + "ignore_missing": true + } + }, { + "remove": { + "field": "nginx.access.info" + } }, { "split": { "field": "nginx.access.remote_ip_list", diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log index e303a6d516d7..d107d245269c 100644 --- a/filebeat/module/nginx/access/test/test.log +++ b/filebeat/module/nginx/access/test/test.log @@ -4,3 +4,4 @@ 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" +127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-" diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 62efe0bb908b..36f5949265c0 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -362,5 +362,51 @@ "name" : "access" } } + }, + { + "_index" : "filebeat-6.0.0-alpha2-2017.05.30", + "_type" : "doc", + "_id" : "AVxWUuZ8OMOtqbaTipsE", + "_score" : 1.0, + "_source": { + "@timestamp": "2018-04-12T07:48:40.000Z", + "nginx": { + "access": { + "body_sent": { + "bytes": "0" + }, + "referrer": "-", + "remote_ip": "127.0.0.1", + "remote_ip_list": [ + "127.0.0.1" + ], + "response_code": "400", + "user_agent": { + "device": "Other", + "name": "Other", + "os": "Other", + "os_name": "Other" + }, + "user_name": "-" + } + }, + "beat" : { + "hostname" : "a-mac-with-esc-key-2.local", + "name" : "a-mac-with-esc-key-2.local", + "version" : "6.0.0-alpha2" + }, + "prospector" : { + "type" : "log" + }, + "input" : { + "type" : "log" + }, + "read_timestamp": "2018-04-13T11:13:43.103Z", + "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log", + "fileset" : { + "module" : "nginx", + "name" : "access" + } + } } ]