From b6194b4354e0acc49078759fc6056452f13647b1 Mon Sep 17 00:00:00 2001
From: Sebastian <sep.gil@gmail.com>
Date: Mon, 29 May 2017 15:00:48 +0200
Subject: [PATCH] Change grok pattern to fetch correct IP from X-Forwarded-For
 list (#4351)

* Change grok pattern to fetch correct IP from X-Forwarded-For list.

* Document change to the nginx module in the changelog.

* Add tests for nginx access log.
---
 CHANGELOG.asciidoc                            |  1 +
 .../module/nginx/access/ingest/default.json   |  2 +-
 filebeat/module/nginx/access/test/test.log    |  1 +
 .../nginx/access/test/test.log-expected.json  | 58 +++++++++++++++++++
 4 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 filebeat/module/nginx/access/test/test.log
 create mode 100644 filebeat/module/nginx/access/test/test.log-expected.json

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index a09852b34e93..d8c7cf79cfa6 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits]
 - Add filebeat.config.path as replacement for config_dir. {pull}4051[4051]
 - Add a `recursive_glob.enabled` setting to expand `**` in patterns. {pull}3980[3980]
 - Add Icinga module. {pull}3904[3904]
+- Add ability to parse nginx logs exposing the X-Forwarded-For header instead of the remote address.
 
 *Heartbeat*
 
diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
index 5f57757267f0..2e7a3d88ba08 100644
--- a/filebeat/module/nginx/access/ingest/default.json
+++ b/filebeat/module/nginx/access/ingest/default.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "%{IPORHOST:nginx.access.remote_ip} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
+        "%{IPORHOST:nginx.access.remote_ip}(,\\s%{IPORHOST})* - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
         ],
       "ignore_missing": true
     }
diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log
new file mode 100644
index 000000000000..5b5e1a464f67
--- /dev/null
+++ b/filebeat/module/nginx/access/test/test.log
@@ -0,0 +1 @@
+10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
new file mode 100644
index 000000000000..957394db25d8
--- /dev/null
+++ b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -0,0 +1,58 @@
+[
+    {
+        "_index": "filebeat-2016.12.27",
+        "_type": "log",
+        "_id": "AVlBCaYsqYg9cc5KQfcT",
+        "_score": null,
+        "_source": {
+            "@timestamp": "2016-12-07T10:05:07.000Z",
+            "offset": 191,
+            "nginx": {
+                "access": {
+                    "referrer": "-",
+                    "response_code": "200",
+                    "remote_ip": "10.0.0.2",
+                    "method": "GET",
+                    "user_name": "-",
+                    "http_version": "1.1",
+                    "body_sent": {
+                        "bytes": "571"
+                    },
+                    "url": "/ocelot",
+                    "user_agent": {
+                        "major": "49",
+                        "minor": "0",
+                        "os": "Mac OS X 10.12",
+                        "os_minor": "12",
+                        "os_major": "10",
+                        "name": "Firefox",
+                        "os_name": "Mac OS X",
+                        "device": "Other"
+                    }
+                }
+            },
+            "beat": {
+                "hostname": "192-168-0-7.rdsnet.ro",
+                "name": "192-168-0-7.rdsnet.ro",
+                "version": "6.0.0-alpha1"
+            },
+            "read_timestamp": "2016-12-27T15:52:23.304Z",
+            "source": "module/nginx/access/test/test.log",
+            "fields": {
+                "pipeline_id": "nginx-access-with_plugins",
+                "source_type": "nginx-access"
+            },
+            "prospector": {
+                "type": "log"
+            }
+        },
+        "fields": {
+            "@timestamp": [
+                1481105107000
+            ]
+        },
+        "sort": [
+            1481105107000
+        ]
+    }
+]