Make bytes_sent field of type long. Most expressions are optional now

elastic · Dec 5, 2018 · 6737664 · 6737664
1 parent 9bcc9c7
commit 6737664
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 3 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -79,8 +79,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]
 - Added `syslog_host` variable to HAProxy module to allow syslog listener to bind to configured host. {pull}9366[9366]
-- Added support on Traefik for Common Log Format and Combined Log Format mixed {pull}8768[8768]
-- Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768]
 - Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768].
 
 *Heartbeat*

diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json
@@ -5,7 +5,7 @@
             "grok": {
                 "field": "traefik.access.message",
                 "patterns": [
-                    "%{NUMBER:traefik.access.body_sent.bytes:int}( \"%{GREEDYDATA:traefik.access.referrer}\" \"%{DATA:traefik.access.agent}\" %{NUMBER:traefik.access.request_count:int} \"%{DATA:traefik.access.frontend_name}\" \"%{DATA:traefik.access.backend_url}\" %{NUMBER:traefik.access.duration:int}ms)?"
+                    "(?:%{NUMBER:traefik.access.body_sent.bytes:int}|-)( (?:\"%{DATA:traefik.access.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:int}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:int}ms)?)?"
                 ],
                 "ignore_missing": true
             }

diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json
@@ -68,7 +68,10 @@
         "event.module": "traefik", 
         "input.type": "log", 
         "log.offset": 553, 
+        "traefik.access.backend_url": "http://172.19.0.6:14008", 
         "traefik.access.body_sent.bytes": 2814, 
+        "traefik.access.duration": 247, 
+        "traefik.access.frontend_name": "Host-host1-com-0", 
         "traefik.access.geoip.city_name": "Ottawa", 
         "traefik.access.geoip.continent_name": "North America", 
         "traefik.access.geoip.country_iso_code": "CA", 
@@ -79,8 +82,18 @@
         "traefik.access.http_version": "2.0", 
         "traefik.access.method": "GET", 
         "traefik.access.remote_ip": "70.29.80.15", 
+        "traefik.access.request_count": 13, 
         "traefik.access.response_code": "200", 
         "traefik.access.url": "/en/", 
+        "traefik.access.user_agent.device": "iPhone", 
+        "traefik.access.user_agent.major": "11", 
+        "traefik.access.user_agent.minor": "0", 
+        "traefik.access.user_agent.name": "Mobile Safari", 
+        "traefik.access.user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1", 
+        "traefik.access.user_agent.os": "iOS 11.2.5", 
+        "traefik.access.user_agent.os_major": "11", 
+        "traefik.access.user_agent.os_minor": "2", 
+        "traefik.access.user_agent.os_name": "iOS", 
         "traefik.access.user_identifier": "-", 
         "traefik.access.user_name": "-"
     }, 
@@ -118,7 +131,10 @@
         "event.module": "traefik", 
         "input.type": "log", 
         "log.offset": 931, 
+        "traefik.access.backend_url": "http://172.25.0.9:4140", 
         "traefik.access.body_sent.bytes": 85, 
+        "traefik.access.duration": 13, 
+        "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", 
         "traefik.access.geoip.city_name": "Warsaw", 
         "traefik.access.geoip.continent_name": "Europe", 
         "traefik.access.geoip.country_iso_code": "PL", 
@@ -129,8 +145,14 @@
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 
         "traefik.access.remote_ip": "94.254.131.115", 
+        "traefik.access.request_count": 623112, 
         "traefik.access.response_code": "200", 
         "traefik.access.url": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", 
+        "traefik.access.user_agent.device": "Generic Smartphone", 
+        "traefik.access.user_agent.name": "Other", 
+        "traefik.access.user_agent.original": "Android", 
+        "traefik.access.user_agent.os": "Android", 
+        "traefik.access.user_agent.os_name": "Android", 
         "traefik.access.user_identifier": "-", 
         "traefik.access.user_name": "-"
     }, 
@@ -140,7 +162,10 @@
         "event.module": "traefik", 
         "input.type": "log", 
         "log.offset": 1267, 
+        "traefik.access.backend_url": "http://172.25.0.6:4140", 
         "traefik.access.body_sent.bytes": 150, 
+        "traefik.access.duration": 8, 
+        "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", 
         "traefik.access.geoip.city_name": "Katowice", 
         "traefik.access.geoip.continent_name": "Europe", 
         "traefik.access.geoip.country_iso_code": "PL", 
@@ -151,8 +176,14 @@
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 
         "traefik.access.remote_ip": "89.64.35.193", 
+        "traefik.access.request_count": 623114, 
         "traefik.access.response_code": "200", 
         "traefik.access.url": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", 
+        "traefik.access.user_agent.device": "Generic Smartphone", 
+        "traefik.access.user_agent.name": "Other", 
+        "traefik.access.user_agent.original": "Android", 
+        "traefik.access.user_agent.os": "Android", 
+        "traefik.access.user_agent.os_name": "Android", 
         "traefik.access.user_identifier": "-", 
         "traefik.access.user_name": "-"
     },