Change grok pattern to fetch correct IP from X-Forwarded-For list (el…

…astic#4351) * Change grok pattern to fetch correct IP from X-Forwarded-For list. * Document change to the nginx module in the changelog. * Add tests for nginx access log.
tsg · May 29, 2017 · b6194b4 · b6194b4
1 parent 813466d
commit b6194b4
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits]
 - Add filebeat.config.path as replacement for config_dir. {pull}4051[4051]
 - Add a `recursive_glob.enabled` setting to expand `**` in patterns. {pull}3980[3980]
 - Add Icinga module. {pull}3904[3904]
+- Add ability to parse nginx logs exposing the X-Forwarded-For header instead of the remote address.
 
 *Heartbeat*
 

diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{IPORHOST:nginx.access.remote_ip} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
+        "%{IPORHOST:nginx.access.remote_ip}(,\\s%{IPORHOST})* - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
         ],
       "ignore_missing": true
     }

diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log
@@ -0,0 +1 @@
+10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -0,0 +1,58 @@
+[
+    {
+        "_index": "filebeat-2016.12.27",
+        "_type": "log",
+        "_id": "AVlBCaYsqYg9cc5KQfcT",
+        "_score": null,
+        "_source": {
+            "@timestamp": "2016-12-07T10:05:07.000Z",
+            "offset": 191,
+            "nginx": {
+                "access": {
+                    "referrer": "-",
+                    "response_code": "200",
+                    "remote_ip": "10.0.0.2",
+                    "method": "GET",
+                    "user_name": "-",
+                    "http_version": "1.1",
+                    "body_sent": {
+                        "bytes": "571"
+                    },
+                    "url": "/ocelot",
+                    "user_agent": {
+                        "major": "49",
+                        "minor": "0",
+                        "os": "Mac OS X 10.12",
+                        "os_minor": "12",
+                        "os_major": "10",
+                        "name": "Firefox",
+                        "os_name": "Mac OS X",
+                        "device": "Other"
+                    }
+                }
+            },
+            "beat": {
+                "hostname": "192-168-0-7.rdsnet.ro",
+                "name": "192-168-0-7.rdsnet.ro",
+                "version": "6.0.0-alpha1"
+            },
+            "read_timestamp": "2016-12-27T15:52:23.304Z",
+            "source": "module/nginx/access/test/test.log",
+            "fields": {
+                "pipeline_id": "nginx-access-with_plugins",
+                "source_type": "nginx-access"
+            },
+            "prospector": {
+                "type": "log"
+            }
+        },
+        "fields": {
+            "@timestamp": [
+                1481105107000
+            ]
+        },
+        "sort": [
+            1481105107000
+        ]
+    }
+]
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"