Keep unparsed user agent information in user_agent.original (#8537)

Keep unparsed raw user agent information in user_agent.original
elastic · Oct 5, 2018 · f3e0801 · f3e0801
1 parent 7271c55
commit f3e0801
Show file tree

Hide file tree

Showing 15 changed files with 48 additions and 36 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -122,12 +122,12 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff]
 
 - Make inputsource generic taking bufio.SplitFunc as input {pull}7746[7746]
 - Add custom unpack to log hints config to avoid env resolution {pull}7710[7710]
-- Keep raw user agent information after parsing as user_agent_raw in Filebeat modules. {pull}7823[7832]
 - Make docker input check if container strings are empty {pull}7960[7960]
 - Add tag "truncated" to "log.flags" if incoming line is longer than configured limit. {pull}7991[7991]
 - Add tag "multiline" to "log.flags" if event consists of multiple lines. {pull}7997[7997]
 - Add haproxy module. {pull}8014[8014]
 - Release `docker` input as GA. {pull}8328[8328]
+- Keep unparsed user agent information in user_agent.original. {pull}8537[8537]
 - Better tracking of number of open file descriptors. {pull}7986[7986]
 
 *Heartbeat*

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -252,14 +252,16 @@ The name of the operating system.
 
 --
 
-*`apache2.access.user_agent.raw`*::
+*`apache2.access.user_agent.original`*::
 +
 --
 type: text
 
-Raw user agent value before parsing by ingest-user-agent plugin.
+Original user agent value before parsing by ingest-user-agent plugin.
 
 
+Field is not indexed.
+
 --
 
 [float]
@@ -4037,14 +4039,16 @@ The name of the operating system.
 
 --
 
-*`iis.access.user_agent.raw`*::
+*`iis.access.user_agent.original`*::
 +
 --
 type: text
 
-Raw user agent value before parsing by ingest-user-agent plugin.
+Original user agent value before parsing by ingest-user-agent plugin.
 
 
+Field is not indexed.
+
 --
 
 [float]
@@ -5355,14 +5359,16 @@ The name of the operating system.
 
 --
 
-*`nginx.access.user_agent.raw`*::
+*`nginx.access.user_agent.original`*::
 +
 --
 type: text
 
-Raw user agent value before parsing by ingest-user-agent plugin.
+Original user agent value before parsing by ingest-user-agent plugin.
 
 
+Field is not indexed.
+
 --
 
 [float]
@@ -6365,14 +6371,16 @@ The name of the operating system.
 
 --
 
-*`traefik.access.user_agent.raw`*::
+*`traefik.access.user_agent.original`*::
 +
 --
 type: text
 
-Raw user agent value before parsing by ingest-user-agent plugin.
+Original user agent value before parsing by ingest-user-agent plugin.
 
 
+Field is not indexed.
+
 --
 
 [float]

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/apache2/access/_meta/fields.yml b/filebeat/module/apache2/access/_meta/fields.yml
@@ -85,10 +85,11 @@
           type: keyword
           description: >
             The name of the operating system.
-        - name: raw
+        - name: original
           type: text
+          index: false
           description: >
-              Raw user agent value before parsing by ingest-user-agent plugin.
+            Original user agent value before parsing by ingest-user-agent plugin.
     - name: geoip
       type: group
       description: >

diff --git a/filebeat/module/apache2/access/ingest/default.json b/filebeat/module/apache2/access/ingest/default.json
@@ -37,7 +37,7 @@
   }, {
     "rename": {
       "field": "apache2.access.agent",
-      "target_field": "apache2.access.user_agent.raw",
+      "target_field": "apache2.access.user_agent.original",
       "ignore_failure": true
     }
   }, {

diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json
@@ -27,11 +27,11 @@
         "apache2.access.user_agent.major": "50", 
         "apache2.access.user_agent.minor": "0", 
         "apache2.access.user_agent.name": "Firefox", 
+        "apache2.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", 
         "apache2.access.user_agent.os": "Mac OS X 10.12", 
         "apache2.access.user_agent.os_major": "10", 
         "apache2.access.user_agent.os_minor": "12", 
         "apache2.access.user_agent.os_name": "Mac OS X", 
-        "apache2.access.user_agent.raw": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", 
         "apache2.access.user_name": "-", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 
@@ -63,10 +63,10 @@
         "apache2.access.user_agent.major": "15", 
         "apache2.access.user_agent.minor": "0", 
         "apache2.access.user_agent.name": "Firefox Alpha", 
+        "apache2.access.user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", 
         "apache2.access.user_agent.os": "Windows 7", 
         "apache2.access.user_agent.os_name": "Windows 7", 
         "apache2.access.user_agent.patch": "a2", 
-        "apache2.access.user_agent.raw": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", 
         "apache2.access.user_name": "-", 
         "fileset.module": "apache2", 
         "fileset.name": "access", 

diff --git a/filebeat/module/iis/access/_meta/fields.yml b/filebeat/module/iis/access/_meta/fields.yml
@@ -130,10 +130,11 @@
           type: keyword
           description: >
             The name of the operating system.
-        - name: raw
+        - name: original
           type: text
+          index: false
           description: >
-              Raw user agent value before parsing by ingest-user-agent plugin.
+            Original user agent value before parsing by ingest-user-agent plugin.
     - name: geoip
       type: group
       description: >

diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json
@@ -37,7 +37,7 @@
   }, {
     "rename": {
       "field": "iis.access.agent",
-      "target_field": "iis.access.user_agent.raw"
+      "target_field": "iis.access.user_agent.original"
     }
   }, {
     "geoip": {

diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json
@@ -24,9 +24,9 @@
         "iis.access.user_agent.major": "57", 
         "iis.access.user_agent.minor": "0", 
         "iis.access.user_agent.name": "Firefox", 
+        "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_agent.os": "Windows", 
         "iis.access.user_agent.os_name": "Windows", 
-        "iis.access.user_agent.raw": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_name": "-", 
         "iis.access.win32_status": "0", 
         "input.type": "log", 
@@ -55,9 +55,9 @@
         "iis.access.user_agent.major": "57", 
         "iis.access.user_agent.minor": "0", 
         "iis.access.user_agent.name": "Firefox", 
+        "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_agent.os": "Windows", 
         "iis.access.user_agent.os_name": "Windows", 
-        "iis.access.user_agent.raw": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_name": "-", 
         "iis.access.win32_status": "0", 
         "input.type": "log", 
@@ -96,9 +96,9 @@
         "iis.access.user_agent.major": "57", 
         "iis.access.user_agent.minor": "0", 
         "iis.access.user_agent.name": "Firefox", 
+        "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_agent.os": "Windows", 
         "iis.access.user_agent.os_name": "Windows", 
-        "iis.access.user_agent.raw": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", 
         "iis.access.user_name": "-", 
         "iis.access.win32_status": "0", 
         "input.type": "log", 

diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml
@@ -92,10 +92,11 @@
           type: keyword
           description: >
             The name of the operating system.
-        - name: raw
+        - name: original
           type: text
+          index: false
           description: >
-              Raw user agent value before parsing by ingest-user-agent plugin.
+            Original user agent value before parsing by ingest-user-agent plugin.
     - name: geoip
       type: group
       description: >

diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json
@@ -61,7 +61,7 @@
   }, {
     "rename": {
       "field": "nginx.access.agent",
-      "target_field": "nginx.access.user_agent.raw"
+      "target_field": "nginx.access.user_agent.original"
     }
   }, {
     "geoip": {

diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -20,11 +20,11 @@
         "nginx.access.user_agent.major": "49", 
         "nginx.access.user_agent.minor": "0", 
         "nginx.access.user_agent.name": "Firefox", 
+        "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_agent.os": "Mac OS X 10.12", 
         "nginx.access.user_agent.os_major": "10", 
         "nginx.access.user_agent.os_minor": "12", 
         "nginx.access.user_agent.os_name": "Mac OS X", 
-        "nginx.access.user_agent.raw": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_name": "-", 
         "offset": 0, 
         "prospector.type": "log"
@@ -48,10 +48,10 @@
         "nginx.access.user_agent.major": "15", 
         "nginx.access.user_agent.minor": "0", 
         "nginx.access.user_agent.name": "Firefox Alpha", 
+        "nginx.access.user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", 
         "nginx.access.user_agent.os": "Windows 7", 
         "nginx.access.user_agent.os_name": "Windows 7", 
         "nginx.access.user_agent.patch": "a2", 
-        "nginx.access.user_agent.raw": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", 
         "nginx.access.user_name": "-", 
         "offset": 183, 
         "prospector.type": "log"
@@ -84,11 +84,11 @@
         "nginx.access.user_agent.major": "49", 
         "nginx.access.user_agent.minor": "0", 
         "nginx.access.user_agent.name": "Firefox", 
+        "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_agent.os": "Mac OS X 10.12", 
         "nginx.access.user_agent.os_major": "10", 
         "nginx.access.user_agent.os_minor": "12", 
         "nginx.access.user_agent.os_name": "Mac OS X", 
-        "nginx.access.user_agent.raw": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_name": "-", 
         "offset": 341, 
         "prospector.type": "log"
@@ -119,11 +119,11 @@
         "nginx.access.user_agent.major": "49", 
         "nginx.access.user_agent.minor": "0", 
         "nginx.access.user_agent.name": "Firefox", 
+        "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_agent.os": "Mac OS X 10.12", 
         "nginx.access.user_agent.os_major": "10", 
         "nginx.access.user_agent.os_minor": "12", 
         "nginx.access.user_agent.os_name": "Mac OS X", 
-        "nginx.access.user_agent.raw": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", 
         "nginx.access.user_name": "-", 
         "offset": 527, 
         "prospector.type": "log"
@@ -155,9 +155,9 @@
         "nginx.access.url": "/assets/xxxx?q=100", 
         "nginx.access.user_agent.device": "Other", 
         "nginx.access.user_agent.name": "Other", 
+        "nginx.access.user_agent.original": "Amazon CloudFront", 
         "nginx.access.user_agent.os": "Other", 
         "nginx.access.user_agent.os_name": "Other", 
-        "nginx.access.user_agent.raw": "Amazon CloudFront", 
         "nginx.access.user_name": "-", 
         "offset": 693, 
         "prospector.type": "log"
@@ -187,9 +187,9 @@
         "nginx.access.user_agent.major": "1", 
         "nginx.access.user_agent.minor": "0", 
         "nginx.access.user_agent.name": "Facebot", 
+        "nginx.access.user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", 
         "nginx.access.user_agent.os": "Other", 
         "nginx.access.user_agent.os_name": "Other", 
-        "nginx.access.user_agent.raw": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", 
         "nginx.access.user_name": "-", 
         "offset": 845, 
         "prospector.type": "log"
@@ -208,9 +208,9 @@
         "nginx.access.response_code": "400", 
         "nginx.access.user_agent.device": "Other", 
         "nginx.access.user_agent.name": "Other", 
+        "nginx.access.user_agent.original": "-", 
         "nginx.access.user_agent.os": "Other", 
         "nginx.access.user_agent.os_name": "Other", 
-        "nginx.access.user_agent.raw": "-", 
         "nginx.access.user_name": "-", 
         "offset": 1085, 
         "prospector.type": "log"

diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml
@@ -85,10 +85,11 @@
           type: keyword
           description: >
             The name of the operating system.
-        - name: raw
+        - name: original
           type: text
+          index: false
           description: >
-              Raw user agent value before parsing by ingest-user-agent plugin.
+            Original user agent value before parsing by ingest-user-agent plugin.
     - name: geoip
       type: group
       description: >

diff --git a/filebeat/module/traefik/access/ingest/pipeline.json b/filebeat/module/traefik/access/ingest/pipeline.json
@@ -50,7 +50,7 @@
         {
             "rename": {
                 "field": "traefik.access.agent",
-                "target_field": "traefik.access.user_agent.raw"
+                "target_field": "traefik.access.user_agent.original"
             }
         },
         {

diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json
@@ -17,10 +17,10 @@
         "traefik.access.user_agent.major": "61", 
         "traefik.access.user_agent.minor": "0", 
         "traefik.access.user_agent.name": "Chrome", 
+        "traefik.access.user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", 
         "traefik.access.user_agent.os": "Linux", 
         "traefik.access.user_agent.os_name": "Linux", 
         "traefik.access.user_agent.patch": "3163", 
-        "traefik.access.user_agent.raw": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", 
         "traefik.access.user_name": "-"
     }, 
     {
@@ -48,10 +48,10 @@
         "traefik.access.user_agent.major": "61", 
         "traefik.access.user_agent.minor": "0", 
         "traefik.access.user_agent.name": "Chrome", 
+        "traefik.access.user_agent.original": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", 
         "traefik.access.user_agent.os": "Linux", 
         "traefik.access.user_agent.os_name": "Linux", 
         "traefik.access.user_agent.patch": "3163", 
-        "traefik.access.user_agent.raw": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36", 
         "traefik.access.user_name": "-"
     }
 ]