Support log_format combined setting of NGINX access logs (#6858)

Support `log_format combined` setting of nginx access logs
elastic · Apr 16, 2018 · 13fca49 · 13fca49
1 parent e611c9d
commit 13fca49
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -122,6 +122,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Make registry file permission configurable. {pull}6455[6455]
 - Add MongoDB module. {pull}6283[6238]
 - Add Ingest pipeline loading to setup. {pull}6814[6814]
+- Add support of log_format combined to NGINX access logs. {pull}6858[6858]
 
 *Heartbeat*
 

diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -4,13 +4,26 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
+        "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\""
         ],
       "pattern_definitions": {
         "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*"
       },
       "ignore_missing": true
     }
+  }, {
+    "grok": {
+      "field": "nginx.access.info",
+      "patterns": [
+          "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}",
+          ""
+      ],
+      "ignore_missing": true
+    }
+  }, {
+    "remove": {
+      "field": "nginx.access.info"
+    }
   }, {
     "split": {
       "field": "nginx.access.remote_ip_list",

diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log
@@ -4,3 +4,4 @@
 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0"
 "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront"
 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)"
+127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-"
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -362,5 +362,51 @@
         "name" : "access"
       }
     }
+  },
+  {
+    "_index" : "filebeat-6.0.0-alpha2-2017.05.30",
+    "_type" : "doc",
+    "_id" : "AVxWUuZ8OMOtqbaTipsE",
+    "_score" : 1.0,
+    "_source": {
+      "@timestamp": "2018-04-12T07:48:40.000Z",
+      "nginx": {
+      "access": {
+        "body_sent": {
+          "bytes": "0"
+        },
+        "referrer": "-",
+        "remote_ip": "127.0.0.1",
+        "remote_ip_list": [
+          "127.0.0.1"
+        ],
+        "response_code": "400",
+        "user_agent": {
+         "device": "Other",
+         "name": "Other",
+         "os": "Other",
+         "os_name": "Other"
+        },
+        "user_name": "-"
+      }
+      },
+      "beat" : {
+        "hostname" : "a-mac-with-esc-key-2.local",
+        "name" : "a-mac-with-esc-key-2.local",
+        "version" : "6.0.0-alpha2"
+      },
+      "prospector" : {
+        "type" : "log"
+      },
+      "input" : {
+        "type" : "log"
+      },
+      "read_timestamp": "2018-04-13T11:13:43.103Z",
+      "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/nginx/access/test/test.log",
+      "fileset" : {
+        "module" : "nginx",
+        "name" : "access"
+      }
+    }
   }
 ]