[Filebeat] Preserve case of http.request.method (elastic#18359)

* Preserve case of http.request.method ECS previously specified normalizing http.request.method to lowercase. This resulted in the loss of information. Affects filesets from the following versions: - apache/access (7.7 - 7.8) - elasticsearch/audit (7.7 - 7.8) - iis/access (7.7 - 7.8) - iis/error (7.7 - 7.8) - nginx/access (7.8) - nginx/ingress_controller (7.8) - aws/elb (7.7 - 7.8) - suricata/eve (7.4 - 7.8) - zeek/http (7.8) Closes elastic#18154
leehinman · May 27, 2020 · 87c3ad3 · 87c3ad3
1 parent 306a5a8
commit 87c3ad3
Show file tree

Hide file tree

Showing 35 changed files with 153 additions and 177 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -24,7 +24,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 - Improve ECS field mappings in panw module.  event.outcome now only contains success/failure per ECS specification. {issue}16025[16025] {pull}17910[17910]
-- Improve ECS categorization field mappings for nginx module. http.request.referrer is now lowercase & http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
+- Improve ECS categorization field mappings for nginx module. http.request.referrer only populated when nginx sets a value {issue}16174[16174] {pull}17844[17844]
 - Improve ECS field mappings in santa module. move hash.sha256 to process.hash.sha256 & move certificate fields to santa.certificate . {issue}16180[16180] {pull}17982[17982]
 - With the default configuration the cloud modules (aws, azure, googlecloud, o365, okta)
 will no longer send the `host` field that contains information about the host Filebeat is
@@ -33,6 +33,7 @@ happened. {issue}13920[13920] {pull}18223[18223]
 - With the default configuration the cef and panw modules will no longer send the `host`
 field. You can revert this change by configuring tags for the module and omitting
 `forwarded` from the list. {issue}13920[13920] {pull}18223[18223]
+- Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
 
 *Heartbeat*
 

diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml
@@ -34,9 +34,6 @@ processors:
     field: event.outcome
     value: failure
     if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"
-- lowercase:
-    field: http.request.method
-    ignore_missing: true
 - grok:
     field: source.address
     ignore_missing: true

diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json
@@ -7,7 +7,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 45,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -27,7 +27,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 209,
         "http.response.status_code": 404,
         "http.version": "1.1",
@@ -63,7 +63,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 45,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -92,7 +92,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 206,
         "http.response.status_code": 404,
         "http.version": "1.1",
@@ -121,7 +121,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 201,
         "http.response.status_code": 404,
         "http.version": "1.1",

diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json
@@ -8,7 +8,7 @@
         "event.kind": "event",
         "event.module": "apache",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 1375,
         "http.version": "1.1",
         "input.type": "log",
@@ -30,7 +30,7 @@
         "event.kind": "event",
         "event.module": "apache",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.version": "1.1",
         "input.type": "log",
         "log.offset": 276,

diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json
@@ -8,7 +8,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 499,
         "http.response.status_code": 404,

diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json
@@ -7,7 +7,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 209,
         "http.response.status_code": 404,
         "http.version": "1.1",
@@ -27,7 +27,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 499,
         "http.response.status_code": 404,
@@ -71,7 +71,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 612,
         "http.response.status_code": 404,
@@ -99,7 +99,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 612,
         "http.response.status_code": 200,

diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json
@@ -7,7 +7,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 491,
         "http.response.status_code": 200,
@@ -33,7 +33,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 484,
         "http.response.status_code": 200,
@@ -61,7 +61,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "http://192.168.33.72/",
         "http.response.body.bytes": 504,
         "http.response.status_code": 404,
@@ -89,7 +89,7 @@
         "event.module": "apache",
         "event.outcome": "success",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 484,
         "http.response.status_code": 200,
@@ -117,7 +117,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 504,
         "http.response.status_code": 404,
@@ -145,7 +145,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 504,
         "http.response.status_code": 404,
@@ -173,7 +173,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 498,
         "http.response.status_code": 404,
@@ -201,7 +201,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 499,
         "http.response.status_code": 404,
@@ -229,7 +229,7 @@
         "event.module": "apache",
         "event.outcome": "failure",
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.request.referrer": "-",
         "http.response.body.bytes": 499,
         "http.response.status_code": 404,

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml
@@ -40,9 +40,6 @@ processors:
         ctx.event.outcome = 'failure';
       }
 
-- lowercase:
-    field: http.request.method
-    ignore_missing: true
 - set:
     field: host.id
     value: "{{elasticsearch.node.id}}"

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
@@ -13,7 +13,7 @@
         "event.outcome": "failure",
         "fileset.name": "audit",
         "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "input.type": "log",
         "log.offset": 0,
         "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,102+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"pkduyMB5Tly6xgmkYbZi-A\"}",
@@ -37,7 +37,7 @@
         "event.outcome": "failure",
         "fileset.name": "audit",
         "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "input.type": "log",
         "log.offset": 690,
         "message": "{\"type\": \"audit\", \"timestamp\":\"2019-06-11T15:03:32,778+0000\", \"node.id\":\"Xaq2BFVcQ1OhyMrjL8gNOg\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"user.name\":\"elastic\", \"origin.type\":\"rest\", \"origin.address\":\"172.17.0.1:40380\", \"url.path\":\"/\", \"request.method\":\"GET\", \"request.id\":\"KPgEINaXSbGNaIobp8OcMw\"}",

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit.log-expected.json
@@ -202,7 +202,7 @@
         "fileset.name": "audit",
         "host.id": "y8fa3M5zSSGo1M_KJRMUXw",
         "http.request.body.content": "\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "input.type": "log",
         "log.offset": 2056,
         "message": "{\"@timestamp\":\"2019-01-27T20:15:10,380\", \"node.name\":\"node-0\", \"node.id\":\"y8fa3M5zSSGo1M_KJRMUXw\", \"event.type\":\"rest\", \"event.action\":\"authentication_success\", \"user.name\":\"elastic-admin\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:58955\", \"realm\":\"default_file\", \"url.path\":\"/_search\", \"request.method\":\"GET\", \"request.body\":\"\\n{\\n    \\\"query\\\" : {\\n        \\\"term\\\" : { \\\"user\\\" : \\\"kimchy\\\" }\\n    }\\n}\\n\", \"request.id\":\"WzL_kb6VSvOhAq0twPvHOQ\"}",

diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml
@@ -110,9 +110,6 @@ processors:
     field: event.type
     value: connection
     if: "ctx?.source?.ip != null && ctx?.destination?.ip != null"
-- lowercase:
-    field: http.request.method
-    ignore_missing: true
 - append:
     field: related.ip
     value: "{{source.ip}}"

diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json
@@ -17,7 +17,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 404,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 64,
@@ -58,7 +58,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 404,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 2,
@@ -99,7 +99,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 401,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -139,7 +139,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 401,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -179,7 +179,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 404,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 64,

diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json
@@ -17,7 +17,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 404,
         "iis.access.sub_status": 4,
         "iis.access.win32_status": 2,
@@ -57,7 +57,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 200,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -90,7 +90,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 200,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -123,7 +123,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 200,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,

diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json
@@ -19,7 +19,7 @@
         ],
         "fileset.name": "access",
         "http.request.body.bytes": 456,
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 123,
         "http.response.status_code": 200,
         "http.version": "1.1",

diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json
@@ -17,7 +17,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 200,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -63,7 +63,7 @@
         "event.outcome": "success",
         "fileset.name": "access",
         "http.request.body.bytes": 456,
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 123,
         "http.response.status_code": 200,
         "iis.access.site_name": "W3SVC1",
@@ -106,7 +106,7 @@
         ],
         "fileset.name": "access",
         "http.request.body.bytes": 456,
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.body.bytes": 123,
         "http.response.status_code": 200,
         "http.version": "1.1",
@@ -159,7 +159,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 401,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 0,
@@ -200,7 +200,7 @@
             "connection"
         ],
         "fileset.name": "access",
-        "http.request.method": "get",
+        "http.request.method": "GET",
         "http.response.status_code": 404,
         "iis.access.sub_status": 0,
         "iis.access.win32_status": 2,