Change grok pattern to fetch correct IP from X-Forwarded-For list (el…

…astic#4351) * Change grok pattern to fetch correct IP from X-Forwarded-For list. * Document change to the nginx module in the changelog. * Add tests for nginx access log. (cherry picked from commit b6194b4)
tsg · Jul 19, 2017 · 5651279 · 5651279
1 parent faa5577
commit 5651279
Show file tree

Hide file tree

Showing 3 changed files with 351 additions and 0 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -72,6 +72,7 @@ https://github.com/elastic/beats/compare/v5.4.1...master[Check the HEAD diff]
 *Filebeat*
 
 - Add support for loading Xpack Machine Learning configurations from the modules, and added sample configurations for the Nginx module. {pull}4506[4506] {pull}4609[4609]
+-  Add ability to parse nginx logs exposing the X-Forwarded-For header instead of the remote address. {pull}4351[4351]
 
 *Heartbeat*
 

diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log
@@ -0,0 +1,6 @@
+10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-"
+10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
+"10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront"
+2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)"
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -0,0 +1,344 @@
+[
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsE",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2016-12-07T10:05:07.000Z",
+      "offset" : 527,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "200",
+          "remote_ip" : "85.181.35.98",
+          "geoip" : {
+            "continent_name" : "Europe",
+            "country_iso_code" : "DE",
+            "location" : {
+              "lon" : 9.0,
+              "lat" : 51.0
+            }
+          },
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "571"
+          },
+          "remote_ip_list" : [
+            "10.0.0.2",
+            "10.0.0.1",
+            "85.181.35.98"
+          ],
+          "url" : "/ocelot",
+          "user_agent" : {
+            "major" : "49",
+            "minor" : "0",
+            "os" : "Mac OS X 10.12",
+            "os_minor" : "12",
+            "os_major" : "10",
+            "name" : "Firefox",
+            "os_name" : "Mac OS X",
+            "device" : "Other"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.246Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsD",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2017-05-29T19:02:48.000Z",
+      "offset" : 341,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "404",
+          "remote_ip" : "172.17.0.1",
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "612"
+          },
+          "remote_ip_list" : [
+            "172.17.0.1"
+          ],
+          "url" : "/stringpatch",
+          "user_agent" : {
+            "patch" : "a2",
+            "major" : "15",
+            "minor" : "0",
+            "os" : "Windows 7",
+            "name" : "Firefox Alpha",
+            "os_name" : "Windows 7",
+            "device" : "Other"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.246Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsF",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2016-12-07T10:05:07.000Z",
+      "offset" : 693,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "200",
+          "remote_ip" : "85.181.35.98",
+          "geoip" : {
+            "continent_name" : "Europe",
+            "country_iso_code" : "DE",
+            "location" : {
+              "lon" : 9.0,
+              "lat" : 51.0
+            }
+          },
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "571"
+          },
+          "remote_ip_list" : [
+            "85.181.35.98"
+          ],
+          "url" : "/ocelot",
+          "user_agent" : {
+            "major" : "49",
+            "minor" : "0",
+            "os" : "Mac OS X 10.12",
+            "os_minor" : "12",
+            "os_major" : "10",
+            "name" : "Firefox",
+            "os_name" : "Mac OS X",
+            "device" : "Other"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.246Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsC",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2016-12-07T10:05:07.000Z",
+      "offset" : 183,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "200",
+          "remote_ip" : "10.0.0.2",
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "571"
+          },
+          "remote_ip_list" : [
+            "10.0.0.2",
+            "10.0.0.1",
+            "127.0.0.1"
+          ],
+          "url" : "/ocelot",
+          "user_agent" : {
+            "major" : "49",
+            "minor" : "0",
+            "os" : "Mac OS X 10.12",
+            "os_minor" : "12",
+            "os_major" : "10",
+            "name" : "Firefox",
+            "os_name" : "Mac OS X",
+            "device" : "Other"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.245Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsG",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2016-01-22T13:18:29.000Z",
+      "offset" : 845,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "200",
+          "remote_ip" : "199.96.1.1",
+          "geoip" : {
+            "continent_name" : "North America",
+            "city_name" : "Springfield",
+            "country_iso_code" : "US",
+            "region_name" : "Illinois",
+            "location" : {
+              "lon" : -89.6859,
+              "lat" : 39.772
+            }
+          },
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "25507"
+          },
+          "remote_ip_list" : [
+            "10.5.102.222",
+            "199.96.1.1",
+            "204.246.1.1",
+            "10.2.1.185"
+          ],
+          "url" : "/assets/xxxx?q=100",
+          "user_agent" : {
+            "os" : "Other",
+            "name" : "Other",
+            "os_name" : "Other",
+            "device" : "Other"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.246Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtQBaTipsH",
+    "_score" : 1.0,
+    "_source" : {
+      "@timestamp" : "2016-12-30T06:47:09.000Z",
+      "offset" : 1085,
+      "nginx" : {
+        "access" : {
+          "referrer" : "-",
+          "response_code" : "404",
+          "remote_ip" : "2a03:0000:10ff:f00f:0000:0000:0:8000",
+          "geoip" : {
+            "continent_name" : "Europe",
+            "country_iso_code" : "PT",
+            "location" : {
+              "lon" : -8.13057,
+              "lat" : 39.6945
+            }
+          },
+          "method" : "GET",
+          "user_name" : "-",
+          "http_version" : "1.1",
+          "body_sent" : {
+            "bytes" : "8571"
+          },
+          "remote_ip_list" : [
+            "2a03:0000:10ff:f00f:0000:0000:0:8000",
+            "10.225.192.17",
+            "10.2.2.121"
+          ],
+          "url" : "/test.html",
+          "user_agent" : {
+            "major" : "1",
+            "minor" : "0",
+            "os" : "Other",
+            "name" : "Facebot",
+            "os_name" : "Other",
+            "device" : "Spider"
+          }
+        }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "read_timestamp" : "2017-05-29T22:28:06.246Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
+  }
+]