From c9535638098d0eaf042f58579f5eb8391cb6e2b7 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Mon, 2 May 2022 14:39:44 +0200 Subject: [PATCH 01/15] Update: Add support for x_forwarded_for headers in apaches access logs This Pr is heavily inpired by the work done in [this PR](https://github.com/elastic/beats/pull/4417) It is adressing [this ER](https://github.com/elastic/enhancements/issues/14402). `Grok` pattern has been updated to match logs starting with a list of IP adresses and store all those ip in `apache.access.remote_ip`. This pattern is heavily insipred by the one in the nginx integration. I also decided to fill a new field `network.forwarded_ip` as it seems to be the perfect fit. --- packages/apache/_dev/deploy/docker/httpd.conf | 2 +- packages/apache/changelog.yml | 5 + .../_dev/test/pipeline/test-access-basic.log | 4 +- .../test-access-basic.log-expected.json | 468 ++++++++++----- .../test-access-darwin.log-expected.json | 340 ++++++----- .../test-access-ssl-request.log-expected.json | 100 ++-- .../test-access-ubuntu.log-expected.json | 551 ++++++++++-------- .../test-access-vhost.log-expected.json | 62 +- .../elasticsearch/ingest_pipeline/default.yml | 61 +- .../apache/data_stream/access/fields/ecs.yml | 2 + .../data_stream/access/fields/fields.yml | 5 + .../test-error-basic.log-expected.json | 168 +++--- .../test-error-darwin.log-expected.json | 44 +- .../test-error-trace.log-expected.json | 28 +- .../test-error-ubuntu.log-expected.json | 186 +++--- packages/apache/docs/README.md | 4 +- packages/apache/manifest.yml | 2 +- 17 files changed, 1208 insertions(+), 824 deletions(-) diff --git a/packages/apache/_dev/deploy/docker/httpd.conf b/packages/apache/_dev/deploy/docker/httpd.conf index f402947317d..7db7bfc70d1 100644 --- a/packages/apache/_dev/deploy/docker/httpd.conf +++ b/packages/apache/_dev/deploy/docker/httpd.conf @@ -281,7 +281,7 @@ LogLevel warn # The following directives define some format nicknames for use with # a CustomLog directive (see below). # - LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index 643198d6722..7d0923d6678 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.55555.0" + changes: + - description: Add support for x_forwarded_for header + type: enhancement + link: https://github.com/elastic/integrations/pull/3249 - version: "1.4.1" changes: - description: Add correct field mapping for event.created diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log index 4e2cbbe7e54..d3e6b494280 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -4,4 +4,6 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" -monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" \ No newline at end of file +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 3335a0c69df..745d93b141b 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -1,260 +1,300 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508515600Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "::1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105740618Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508560800Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "192.168.33.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105743350Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, - "url": { - "path": "/hello", - "original": "/hello" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508589600Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 } }, + "network": { + "forwarded_ip": "::1" + }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105743828Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "172.17.0.1" + ] + } }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508609200Z", + "kind": "event", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "172.17.0.1" }, "source": { "address": "172.17.0.1", "ip": "172.17.0.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105744250Z", - "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/stringpatch", + "path": "/stringpatch" }, "user": { "name": "-" }, - "url": { - "path": "/stringpatch", - "original": "/stringpatch" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": {} }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508627800Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, - "event": { - "ingested": "2021-12-14T14:34:05.105744722Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/status", + "path": "/status" }, "user": { "name": "-" }, - "url": { - "path": "/status", - "original": "/status" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } }, - "@timestamp": "2019-02-02T04:38:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508646100Z", + "kind": "event", + "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "referrer": "-" @@ -266,89 +306,245 @@ "status_code": 408 } }, + "network": { + "forwarded_ip": "127.0.0.1" + }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105745119Z", - "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, + "tags": [ + "preserve_original_event" + ], "user": { "name": "-" }, "user_agent": { - "name": "Other", "device": { "name": "Other" }, + "name": "Other", "original": "-" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": {} }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508664600Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": { + "remote_ip_list": [ + "10.0.0.2", + "10.0.0.1", + "89.160.20.112" + ] + } + }, + "ecs": { + "version": "1.12.0" + }, "event": { - "ingested": "2021-12-14T14:34:05.105745518Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "category": "web", - "kind": "event", "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508683200Z", + "kind": "event", + "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" }, - "user": { - "name": "-" + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "89.160.20.112" }, + "source": { + "address": "89.160.20.112", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.112" + }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "extension": "mp4", - "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": { + "remote_ip_list": [ + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "10.225.192.17", + "10.2.2.121" + ] + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.508701700Z", + "kind": "event", + "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "source": { + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" }, "tags": [ "preserve_original_event" - ] + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index b056c3dbb95..3965e610f80 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -1,306 +1,348 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:28.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979121500Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "::1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093531222Z", - "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979154800Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "::1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093534426Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979172400Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 } }, + "network": { + "forwarded_ip": "::1" + }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093534891Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:35.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979182800Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "89.160.20.156" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093535376Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:41.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979189400Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 206 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "89.160.20.156" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093535764Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/notfound", + "path": "/notfound" }, "user": { "name": "-" - }, - "url": { - "path": "/notfound", - "original": "/notfound" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:23.979200Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 201 }, "status_code": 404 - } + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "89.160.20.156" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093536142Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hmm", + "path": "/hmm" }, "user": { "name": "-" - }, - "url": { - "path": "/hmm", - "original": "/hmm" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 8d05c25b3a5..5bea62020a1 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -1,6 +1,7 @@ { "expected": [ { + "@timestamp": "2018-08-10T07:45:56.000Z", "apache": { "access": { "ssl": { @@ -9,48 +10,51 @@ } } }, - "@timestamp": "2018-08-10T07:45:56.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.051311500Z", + "kind": "event", + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 1375 } - } + }, + "version": "1.1" }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" + "network": { + "forwarded_ip": "172.30.0.119" }, "source": { "address": "172.30.0.119", "ip": "172.30.0.119" }, - "event": { - "ingested": "2021-12-14T14:34:06.744087534Z", - "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/nagiosxi/ajaxhelper.php", "extension": "php", "original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "path": "/nagiosxi/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-10-16T09:53:47.000Z", "apache": { "access": { "ssl": { @@ -59,59 +63,61 @@ } } }, - "@timestamp": "2019-10-16T09:53:47.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.051380200Z", + "kind": "event", + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" + }, "http": { "request": { "method": "GET" }, "version": "1.1" }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" + "network": { + "forwarded_ip": "89.160.20.156" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.744090082Z", - "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/appl/ajaxhelper.php", "extension": "php", "original": "/appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "path": "/appl/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 92c297c4b33..c83d0c7f730 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -1,515 +1,578 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } }, - "@timestamp": "2016-12-26T16:18:09.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.115992800Z", + "kind": "event", + "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 491 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835525800Z", - "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "network": { + "forwarded_ip": "127.0.0.1" + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Wget", "original": "Wget/1.13.4 (linux-gnu)", "os": { "name": "Linux" }, - "device": { - "name": "Other" - }, "version": "1.13.4" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116024800Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835534600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116039200Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "http://192.168.33.72/" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835540100Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116056100Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835543600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116073600Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835548Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116098300Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835553700Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/test", - "original": "/test" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:10.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116111100Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 498 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835559600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/test", + "path": "/test" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/hello", - "original": "/hello" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116126200Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835563600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/crap", - "original": "/crap" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:17.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.116138600Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835568100Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "network": { + "forwarded_ip": "192.168.33.1" + }, + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/crap", + "path": "/crap" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 8b6a8cbbef2..85bf678356b 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -1,61 +1,63 @@ { "expected": [ { - "destination": { - "domain": "vhost1.domaine.fr" - }, - "source": { - "ip": "192.168.33.2" - }, - "url": { - "path": "/hello", - "original": "/hello", - "domain": "vhost1.domaine.fr" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:14.000Z", "apache": { - "access": {} + "access": { + "remote_ip_list": [ + "fr", + "192.168.33.2" + ] + } }, - "@timestamp": "2016-12-26T16:22:14.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-02T12:08:24.211525800Z", + "kind": "event", + "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:33.387841500Z", - "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "fr", + "domain": "fr" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index dadfb3a4939..e477dbd314b 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -16,7 +16,7 @@ processors: - grok: field: event.original patterns: - - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '(?:%{ADDRESS_LIST:apache.access.remote_ip_list}|%{NOTSPACE:source.address}) - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' @@ -30,6 +30,61 @@ processors: %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true + pattern_definitions: + ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + - split: + field: apache.access.remote_ip_list + separator: '"?,?\s+' + ignore_missing: true + - script: + if: ctx.apache?.access?.remote_ip_list != null && ctx.apache.access.remote_ip_list.length > 0 + lang: painless + source: >- + boolean isPrivate(def dot, def ip) { + try { + StringTokenizer tok = new StringTokenizer(ip, dot); + int firstByte = Integer.parseInt(tok.nextToken()); + int secondByte = Integer.parseInt(tok.nextToken()); + if (firstByte == 10) { + return true; + } + if (firstByte == 192 && secondByte == 168) { + return true; + } + if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { + return true; + } + if (firstByte == 127) { + return true; + } + return false; + } + catch (Exception e) { + return false; + } + } + try { + ctx.source.address = null; + if (ctx.apache.access.remote_ip_list == null) { + return; + } + def found = false; + for (def item : ctx.apache.access.remote_ip_list) { + if (!isPrivate(params.dot, item)) { + ctx.source.address = item; + found = true; + break; + } + } + if (!found) { + ctx.source.address = ctx.apache.access.remote_ip_list[0]; + } + } + catch (Exception e) { + ctx.source.address = null; + } + params: + dot: . - uri_parts: field: _tmp.url_orig ignore_failure: true @@ -60,6 +115,10 @@ processors: ignore_missing: true patterns: - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ + - set: + field: network.forwarded_ip + copy_from: source.ip + ignore_empty_value: true - remove: field: event.created ignore_missing: true diff --git a/packages/apache/data_stream/access/fields/ecs.yml b/packages/apache/data_stream/access/fields/ecs.yml index 12993b02683..7488d3b683a 100644 --- a/packages/apache/data_stream/access/fields/ecs.yml +++ b/packages/apache/data_stream/access/fields/ecs.yml @@ -30,6 +30,8 @@ name: log.level - external: ecs name: message +- external: ecs + name: network.forwarded_ip - external: ecs name: process.pid - external: ecs diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index 402fb490e41..96285e61205 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -9,3 +9,8 @@ type: keyword description: | SSL cipher name. + - name: nginx.access + - name: remote_ip_list + type: array + description: | + An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index abd455abe8a..d110905e8d0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -1,170 +1,170 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.491755700Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T14:34:09.255770595Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T14:34:09.255773246Z", - "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", - "type": "info", + "ingested": "2022-05-02T12:08:24.491785800Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 35708, - "thread": { - "id": 4328636416 - } - }, + "@timestamp": "2011-09-09T10:42:29.902+02:00", "apache": { "error": { "module": "core" } }, - "file": { - "path": "/usr/local/apache2/htdocs/favicon.ico" - }, - "@timestamp": "2011-09-09T10:42:29.902+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.491803700Z", + "kind": "event", + "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "process": { + "pid": 35708, + "thread": { + "id": 4328636416 + } + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:09.255773777Z", - "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 15934 - }, + "@timestamp": "2019-06-27T06:58:09.169+02:00", "apache": { "error": { "module": "include" } }, - "@timestamp": "2019-06-27T06:58:09.169+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.491819800Z", + "kind": "event", + "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "timezone": "GMT+2", + "type": "error" + }, "log": { "level": "warn" }, + "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "process": { + "pid": 15934 + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", - "port": 12345, - "ip": "89.160.20.156" - }, - "event": { - "ingested": "2021-12-14T14:34:09.255774189Z", - "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 12345 }, - "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 15400a6c97a..1a3bf55131c 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -1,59 +1,59 @@ { "expected": [ { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "mpm_prefork" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.149405700Z", - "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "category": "web", - "type": "info", + "ingested": "2022-05-02T12:08:24.559334800Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.149429600Z", - "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", - "type": "info", + "ingested": "2022-05-02T12:08:24.559363400Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 7b5f6e1b4b1..6d30fd94708 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -1,33 +1,33 @@ { "expected": [ { - "process": { - "pid": 121591, - "thread": { - "id": 140413273032448 - } - }, + "@timestamp": "2021-10-20T19:20:59.121+02:00", "apache": { "error": { "module": "rewrite" } }, - "@timestamp": "2021-10-20T19:20:59.121+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "trace3" - }, "event": { - "ingested": "2021-12-09T13:30:34.228018700Z", - "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "category": "web", - "type": "info", + "ingested": "2022-05-02T12:08:24.613195200Z", + "kind": "event", + "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "trace3" }, "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "process": { + "pid": 121591, + "thread": { + "id": 140413273032448 + } + }, "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 92c843f5903..7fd91f13852 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -1,23 +1,23 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:17:53.000+02:00", "apache": { "error": {} }, - "@timestamp": "2016-12-26T16:17:53.000+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.283841100Z", - "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "category": "web", - "type": "info", + "ingested": "2022-05-02T12:08:24.666135200Z", + "kind": "event", + "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "tags": [ @@ -25,192 +25,192 @@ ] }, { + "@timestamp": "2016-12-26T16:22:00.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.666160500Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "http": { + "request": { + "referrer": "http://192.168.33.72/" + } + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "tags": [ "preserve_original_event" - ], + ] + }, + { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:00.000+02:00", "ecs": { "version": "1.12.0" }, - "http": { - "request": { - "referrer": "http://192.168.33.72/" - } - }, "event": { - "ingested": "2021-12-09T13:30:34.283849400Z", - "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "category": "web", - "type": "error", + "ingested": "2022-05-02T12:08:24.666176500Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", - "kind": "event" - } - }, - { - "apache": { - "error": {} + "type": "error" }, "file": { "path": "/var/www/favicon.ico" }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", - "ecs": { - "version": "1.12.0" - }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283853Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.666184200Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283857200Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:10.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/test" - }, - "@timestamp": "2016-12-26T16:22:10.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.666189500Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/test" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/test", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283862600Z", - "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/test", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:13.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/hello" - }, - "@timestamp": "2016-12-26T16:22:13.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.666197600Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/hello" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/hello", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283867900Z", - "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/hello", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:17.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/crap" - }, - "@timestamp": "2016-12-26T16:22:17.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-05-02T12:08:24.666207100Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/crap" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/crap", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283873300Z", - "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/crap", "tags": [ "preserve_original_event" ] diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index 2ef8d6fee40..ae0fc3939bc 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,7 +19,8 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.ssl.cipher | SSL cipher name. | keyword | +| apache.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | array | +| apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | @@ -75,6 +76,7 @@ Access logs collects the Apache access logs. | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | | process.pid | Process id. | long | | process.thread.id | Thread ID. | long | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index 68cd2cad7c6..ca07479b200 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: apache title: Apache HTTP Server -version: 1.4.1 +version: 1.5.0 license: basic description: Collect logs and metrics from Apache servers with Elastic Agent. type: integration From 0058ab5ee3d7d87618a01e34b1ab606536edbb9d Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Mon, 2 May 2022 15:18:23 +0200 Subject: [PATCH 02/15] Update: Pr link in the changelogs --- packages/apache/changelog.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index 7d0923d6678..54fd7052868 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -1,9 +1,9 @@ # newer versions go on top -- version: "1.55555.0" +- version: "1.5.0" changes: - description: Add support for x_forwarded_for header type: enhancement - link: https://github.com/elastic/integrations/pull/3249 + link: https://github.com/elastic/integrations/pull/3251 - version: "1.4.1" changes: - description: Add correct field mapping for event.created From 28ed55832c089f5ded7c6770fb162be5cec6068c Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Mon, 2 May 2022 15:43:58 +0200 Subject: [PATCH 03/15] Update: Painless check property exist before referencing it --- .../test-access-basic.log-expected.json | 18 +++++++++--------- .../test-access-darwin.log-expected.json | 12 ++++++------ .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 +++++++++--------- .../test-access-vhost.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 4 ++++ .../test-error-basic.log-expected.json | 8 ++++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +++++++------- 10 files changed, 45 insertions(+), 41 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 745d93b141b..c035996cd1e 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508515600Z", + "ingested": "2022-05-02T13:43:24.918687300Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508560800Z", + "ingested": "2022-05-02T13:43:24.918722900Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508589600Z", + "ingested": "2022-05-02T13:43:24.918742700Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508609200Z", + "ingested": "2022-05-02T13:43:24.918760800Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508627800Z", + "ingested": "2022-05-02T13:43:24.918778100Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508646100Z", + "ingested": "2022-05-02T13:43:24.918795700Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508664600Z", + "ingested": "2022-05-02T13:43:24.918811600Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508683200Z", + "ingested": "2022-05-02T13:43:24.918823600Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.508701700Z", + "ingested": "2022-05-02T13:43:24.918841300Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 3965e610f80..e6036cac2d3 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979121500Z", + "ingested": "2022-05-02T13:43:25.319844800Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979154800Z", + "ingested": "2022-05-02T13:43:25.319904Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979172400Z", + "ingested": "2022-05-02T13:43:25.319917600Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979182800Z", + "ingested": "2022-05-02T13:43:25.319926800Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979189400Z", + "ingested": "2022-05-02T13:43:25.319936200Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:23.979200Z", + "ingested": "2022-05-02T13:43:25.319970600Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 5bea62020a1..8a9dc982f77 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.051311500Z", + "ingested": "2022-05-02T13:43:25.387258300Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.051380200Z", + "ingested": "2022-05-02T13:43:25.387286100Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index c83d0c7f730..4cb538e42a7 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.115992800Z", + "ingested": "2022-05-02T13:43:25.443545400Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116024800Z", + "ingested": "2022-05-02T13:43:25.443576200Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116039200Z", + "ingested": "2022-05-02T13:43:25.443593400Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116056100Z", + "ingested": "2022-05-02T13:43:25.443607Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116073600Z", + "ingested": "2022-05-02T13:43:25.443618200Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116098300Z", + "ingested": "2022-05-02T13:43:25.443632Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116111100Z", + "ingested": "2022-05-02T13:43:25.443644600Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116126200Z", + "ingested": "2022-05-02T13:43:25.443657700Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.116138600Z", + "ingested": "2022-05-02T13:43:25.443668600Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 85bf678356b..08372284c6f 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T12:08:24.211525800Z", + "ingested": "2022-05-02T13:43:25.540229800Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index e477dbd314b..2a5a47718e9 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -64,6 +64,10 @@ processors: } } try { + if (ctx?.source == null){ + Map map = new HashMap(); + ctx.put("source", map); + } ctx.source.address = null; if (ctx.apache.access.remote_ip_list == null) { return; diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index d110905e8d0..2e8f198c746 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.491755700Z", + "ingested": "2022-05-02T13:43:25.779939400Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.491785800Z", + "ingested": "2022-05-02T13:43:25.779969Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.491803700Z", + "ingested": "2022-05-02T13:43:25.779984900Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.491819800Z", + "ingested": "2022-05-02T13:43:25.779999100Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 1a3bf55131c..2a622bb4ad0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.559334800Z", + "ingested": "2022-05-02T13:43:25.841922500Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.559363400Z", + "ingested": "2022-05-02T13:43:25.841952800Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 6d30fd94708..0ecfcddf16a 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.613195200Z", + "ingested": "2022-05-02T13:43:25.892814400Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 7fd91f13852..36efe19ad5f 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666135200Z", + "ingested": "2022-05-02T13:43:25.940082200Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666160500Z", + "ingested": "2022-05-02T13:43:25.940115400Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666176500Z", + "ingested": "2022-05-02T13:43:25.940122800Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666184200Z", + "ingested": "2022-05-02T13:43:25.940126600Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666189500Z", + "ingested": "2022-05-02T13:43:25.940136500Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666197600Z", + "ingested": "2022-05-02T13:43:25.940165900Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T12:08:24.666207100Z", + "ingested": "2022-05-02T13:43:25.940177300Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", From dbbe9a6adb711cf0539ae89ee9ed126f69747cf5 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Mon, 16 May 2022 15:50:52 +0200 Subject: [PATCH 04/15] Update: simplify grok expression It want the expression ot be as lean as possible. I removed the ?: non capturing group - we want to capture it - It was ignored somehow I removed the %{NOTSPACE:source.address} match : - I am not sure why it is was there in the first place --- .../test-access-basic.log-expected.json | 18 +++++++++--------- .../test-access-darwin.log-expected.json | 12 ++++++------ .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 +++++++++--------- .../test-access-vhost.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../test-error-basic.log-expected.json | 8 ++++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +++++++------- 10 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index c035996cd1e..43792d5ea0a 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918687300Z", + "ingested": "2022-05-16T13:50:20.144426284Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918722900Z", + "ingested": "2022-05-16T13:50:20.144497833Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918742700Z", + "ingested": "2022-05-16T13:50:20.144506425Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918760800Z", + "ingested": "2022-05-16T13:50:20.144512629Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918778100Z", + "ingested": "2022-05-16T13:50:20.144518405Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918795700Z", + "ingested": "2022-05-16T13:50:20.144523835Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918811600Z", + "ingested": "2022-05-16T13:50:20.144528815Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918823600Z", + "ingested": "2022-05-16T13:50:20.144533628Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:24.918841300Z", + "ingested": "2022-05-16T13:50:20.144538591Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index e6036cac2d3..f11cdfe6489 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319844800Z", + "ingested": "2022-05-16T13:50:20.447937617Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319904Z", + "ingested": "2022-05-16T13:50:20.447957177Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319917600Z", + "ingested": "2022-05-16T13:50:20.447963182Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319926800Z", + "ingested": "2022-05-16T13:50:20.447967864Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319936200Z", + "ingested": "2022-05-16T13:50:20.447972147Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.319970600Z", + "ingested": "2022-05-16T13:50:20.447976257Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 8a9dc982f77..ee6a6917fbd 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.387258300Z", + "ingested": "2022-05-16T13:50:20.512872057Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.387286100Z", + "ingested": "2022-05-16T13:50:20.512981325Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 4cb538e42a7..f91906abfa0 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443545400Z", + "ingested": "2022-05-16T13:50:20.556041728Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443576200Z", + "ingested": "2022-05-16T13:50:20.556064927Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443593400Z", + "ingested": "2022-05-16T13:50:20.556071813Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443607Z", + "ingested": "2022-05-16T13:50:20.556077429Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443618200Z", + "ingested": "2022-05-16T13:50:20.556082574Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443632Z", + "ingested": "2022-05-16T13:50:20.556087266Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443644600Z", + "ingested": "2022-05-16T13:50:20.556091781Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443657700Z", + "ingested": "2022-05-16T13:50:20.556096077Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.443668600Z", + "ingested": "2022-05-16T13:50:20.556100484Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 08372284c6f..0afea8e9979 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-02T13:43:25.540229800Z", + "ingested": "2022-05-16T13:50:20.640011702Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 2a5a47718e9..b5263792ebf 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -16,7 +16,7 @@ processors: - grok: field: event.original patterns: - - '(?:%{ADDRESS_LIST:apache.access.remote_ip_list}|%{NOTSPACE:source.address}) - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '%{ADDRESS_LIST:apache.access.remote_ip_list} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 2e8f198c746..89c46b9134c 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.779939400Z", + "ingested": "2022-05-16T13:50:20.844252376Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.779969Z", + "ingested": "2022-05-16T13:50:20.844268840Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.779984900Z", + "ingested": "2022-05-16T13:50:20.844272609Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.779999100Z", + "ingested": "2022-05-16T13:50:20.844275889Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 2a622bb4ad0..0330bc62fc9 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.841922500Z", + "ingested": "2022-05-16T13:50:20.901636128Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.841952800Z", + "ingested": "2022-05-16T13:50:20.901652134Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 0ecfcddf16a..d3cafb05cd6 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.892814400Z", + "ingested": "2022-05-16T13:50:20.951402491Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 36efe19ad5f..58d1e82fba0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940082200Z", + "ingested": "2022-05-16T13:50:20.996170180Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940115400Z", + "ingested": "2022-05-16T13:50:20.996182370Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940122800Z", + "ingested": "2022-05-16T13:50:20.996185740Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940126600Z", + "ingested": "2022-05-16T13:50:20.996188617Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940136500Z", + "ingested": "2022-05-16T13:50:20.996191281Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940165900Z", + "ingested": "2022-05-16T13:50:20.996193923Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-02T13:43:25.940177300Z", + "ingested": "2022-05-16T13:50:20.996196616Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", From 986bd0261cbca722cd854babed882346df6c6cb3 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 17 May 2022 14:06:43 +0200 Subject: [PATCH 05/15] Update: x-forwarded-for match only ip As x-forwarded-for mostly send ip, grok pattern is updated, to match only ip. To improve searchability remote_ip type is set to ip --- .../test-access-basic.log-expected.json | 18 +++++++++--------- .../test-access-darwin.log-expected.json | 12 ++++++------ .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 +++++++++--------- .../test-access-vhost.log-expected.json | 10 ++++++---- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/access/fields/fields.yml | 2 +- .../test-error-basic.log-expected.json | 8 ++++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +++++++------- packages/apache/docs/README.md | 2 +- 12 files changed, 49 insertions(+), 47 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 43792d5ea0a..a547cb3d79e 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144426284Z", + "ingested": "2022-05-17T11:55:26.233486922Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144497833Z", + "ingested": "2022-05-17T11:55:26.233524144Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144506425Z", + "ingested": "2022-05-17T11:55:26.233533112Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144512629Z", + "ingested": "2022-05-17T11:55:26.233539546Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144518405Z", + "ingested": "2022-05-17T11:55:26.233545345Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144523835Z", + "ingested": "2022-05-17T11:55:26.233550730Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144528815Z", + "ingested": "2022-05-17T11:55:26.233556290Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144533628Z", + "ingested": "2022-05-17T11:55:26.233561600Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.144538591Z", + "ingested": "2022-05-17T11:55:26.233567109Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index f11cdfe6489..2331cebe2fc 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447937617Z", + "ingested": "2022-05-17T11:55:26.579482034Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447957177Z", + "ingested": "2022-05-17T11:55:26.579540282Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447963182Z", + "ingested": "2022-05-17T11:55:26.579547385Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447967864Z", + "ingested": "2022-05-17T11:55:26.579553021Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447972147Z", + "ingested": "2022-05-17T11:55:26.579557997Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.447976257Z", + "ingested": "2022-05-17T11:55:26.579660991Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index ee6a6917fbd..072b669c890 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.512872057Z", + "ingested": "2022-05-17T11:55:26.643413415Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.512981325Z", + "ingested": "2022-05-17T11:55:26.643521496Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index f91906abfa0..9432d801ab5 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556041728Z", + "ingested": "2022-05-17T11:55:26.699762495Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556064927Z", + "ingested": "2022-05-17T11:55:26.699787059Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556071813Z", + "ingested": "2022-05-17T11:55:26.699793350Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556077429Z", + "ingested": "2022-05-17T11:55:26.699798378Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556082574Z", + "ingested": "2022-05-17T11:55:26.699803031Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556087266Z", + "ingested": "2022-05-17T11:55:26.699807333Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556091781Z", + "ingested": "2022-05-17T11:55:26.699811536Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556096077Z", + "ingested": "2022-05-17T11:55:26.699815683Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.556100484Z", + "ingested": "2022-05-17T11:55:26.699819831Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 0afea8e9979..b712f7520c1 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -5,7 +5,6 @@ "apache": { "access": { "remote_ip_list": [ - "fr", "192.168.33.2" ] } @@ -16,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-16T13:50:20.640011702Z", + "ingested": "2022-05-17T11:55:26.791300855Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -34,9 +33,12 @@ }, "version": "1.1" }, + "network": { + "forwarded_ip": "192.168.33.2" + }, "source": { - "address": "fr", - "domain": "fr" + "address": "192.168.33.2", + "ip": "192.168.33.2" }, "tags": [ "preserve_original_event" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index b5263792ebf..d74c6791692 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -31,7 +31,7 @@ processors: HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true pattern_definitions: - ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* + ADDRESS_LIST: (%{IP})("?,?\s*(%{IP}))* - split: field: apache.access.remote_ip_list separator: '"?,?\s+' diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index 96285e61205..a2ba06392dd 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -11,6 +11,6 @@ SSL cipher name. - name: nginx.access - name: remote_ip_list - type: array + type: ip description: | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 89c46b9134c..d92e8799721 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.844252376Z", + "ingested": "2022-05-17T11:55:27.022836600Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.844268840Z", + "ingested": "2022-05-17T11:55:27.022851324Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.844272609Z", + "ingested": "2022-05-17T11:55:27.022855172Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.844275889Z", + "ingested": "2022-05-17T11:55:27.022858455Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 0330bc62fc9..1155e879c02 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.901636128Z", + "ingested": "2022-05-17T11:55:27.089682252Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.901652134Z", + "ingested": "2022-05-17T11:55:27.089852681Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index d3cafb05cd6..22cb7ffcaaa 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.951402491Z", + "ingested": "2022-05-17T11:55:27.161512045Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 58d1e82fba0..87d88743cd6 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996170180Z", + "ingested": "2022-05-17T11:55:27.236578729Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996182370Z", + "ingested": "2022-05-17T11:55:27.236599984Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996185740Z", + "ingested": "2022-05-17T11:55:27.236605398Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996188617Z", + "ingested": "2022-05-17T11:55:27.236611271Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996191281Z", + "ingested": "2022-05-17T11:55:27.236617045Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996193923Z", + "ingested": "2022-05-17T11:55:27.236621994Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-16T13:50:20.996196616Z", + "ingested": "2022-05-17T11:55:27.236627563Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index ae0fc3939bc..cece7f9753c 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,7 +19,7 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | array | +| apache.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | ip | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | From 5904f1556c0eaadce472927becb00c3d9558d5a7 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 17 May 2022 14:28:53 +0200 Subject: [PATCH 06/15] Update: migrate ip adress string analysis to CIDR --- .../test-access-basic.log-expected.json | 18 +++++----- .../test-access-darwin.log-expected.json | 12 +++---- .../test-access-ssl-request.log-expected.json | 4 +-- .../test-access-ubuntu.log-expected.json | 18 +++++----- .../test-access-vhost.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 35 ++++++------------- .../test-error-basic.log-expected.json | 8 ++--- .../test-error-darwin.log-expected.json | 4 +-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 ++++---- 10 files changed, 51 insertions(+), 66 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index a547cb3d79e..c60a390015b 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233486922Z", + "ingested": "2022-05-17T12:27:52.752676143Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233524144Z", + "ingested": "2022-05-17T12:27:52.752710961Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233533112Z", + "ingested": "2022-05-17T12:27:52.752720875Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233539546Z", + "ingested": "2022-05-17T12:27:52.752728496Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233545345Z", + "ingested": "2022-05-17T12:27:52.752735179Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233550730Z", + "ingested": "2022-05-17T12:27:52.752741297Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233556290Z", + "ingested": "2022-05-17T12:27:52.752747993Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233561600Z", + "ingested": "2022-05-17T12:27:52.752753866Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.233567109Z", + "ingested": "2022-05-17T12:27:52.752759288Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 2331cebe2fc..06c8479b721 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579482034Z", + "ingested": "2022-05-17T12:27:53.131140454Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579540282Z", + "ingested": "2022-05-17T12:27:53.131166030Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579547385Z", + "ingested": "2022-05-17T12:27:53.131173502Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579553021Z", + "ingested": "2022-05-17T12:27:53.131179495Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579557997Z", + "ingested": "2022-05-17T12:27:53.131184861Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.579660991Z", + "ingested": "2022-05-17T12:27:53.131190159Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 072b669c890..782f00924b7 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.643413415Z", + "ingested": "2022-05-17T12:27:53.199618007Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.643521496Z", + "ingested": "2022-05-17T12:27:53.199639521Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 9432d801ab5..13c741964d6 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699762495Z", + "ingested": "2022-05-17T12:27:53.261379275Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699787059Z", + "ingested": "2022-05-17T12:27:53.261402676Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699793350Z", + "ingested": "2022-05-17T12:27:53.261409439Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699798378Z", + "ingested": "2022-05-17T12:27:53.261414730Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699803031Z", + "ingested": "2022-05-17T12:27:53.261419475Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699807333Z", + "ingested": "2022-05-17T12:27:53.261424008Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699811536Z", + "ingested": "2022-05-17T12:27:53.261428598Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699815683Z", + "ingested": "2022-05-17T12:27:53.261453917Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.699819831Z", + "ingested": "2022-05-17T12:27:53.261494716Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index b712f7520c1..e7cd3d107f5 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T11:55:26.791300855Z", + "ingested": "2022-05-17T12:27:53.353021263Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index d74c6791692..d06b065ca9d 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -39,30 +39,17 @@ processors: - script: if: ctx.apache?.access?.remote_ip_list != null && ctx.apache.access.remote_ip_list.length > 0 lang: painless + tag: Get source address + description: Extract from remote_ip_list, the first non-private IP to source.address source: >- - boolean isPrivate(def dot, def ip) { - try { - StringTokenizer tok = new StringTokenizer(ip, dot); - int firstByte = Integer.parseInt(tok.nextToken()); - int secondByte = Integer.parseInt(tok.nextToken()); - if (firstByte == 10) { - return true; - } - if (firstByte == 192 && secondByte == 168) { - return true; - } - if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { - return true; - } - if (firstByte == 127) { - return true; - } - return false; - } - catch (Exception e) { - return false; - } + boolean isPrivateCIDR(def ip) { + CIDR class_a_network = new CIDR('10.0.0.0/8'); + CIDR class_b_network = new CIDR('172.16.0.0/12'); + CIDR class_c_network = new CIDR('192.168.0.0/16'); + + return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip); } + try { if (ctx?.source == null){ Map map = new HashMap(); @@ -74,7 +61,7 @@ processors: } def found = false; for (def item : ctx.apache.access.remote_ip_list) { - if (!isPrivate(params.dot, item)) { + if (!isPrivateCIDR(item)) { ctx.source.address = item; found = true; break; @@ -87,8 +74,6 @@ processors: catch (Exception e) { ctx.source.address = null; } - params: - dot: . - uri_parts: field: _tmp.url_orig ignore_failure: true diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index d92e8799721..25526d797c7 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.022836600Z", + "ingested": "2022-05-17T12:27:53.604423694Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.022851324Z", + "ingested": "2022-05-17T12:27:53.604440164Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.022855172Z", + "ingested": "2022-05-17T12:27:53.604444992Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.022858455Z", + "ingested": "2022-05-17T12:27:53.604448848Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 1155e879c02..598487159db 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.089682252Z", + "ingested": "2022-05-17T12:27:53.670123707Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.089852681Z", + "ingested": "2022-05-17T12:27:53.670141748Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 22cb7ffcaaa..f8939240a46 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.161512045Z", + "ingested": "2022-05-17T12:27:53.722420070Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 87d88743cd6..c3f5cb43fe5 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236578729Z", + "ingested": "2022-05-17T12:27:53.771795455Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236599984Z", + "ingested": "2022-05-17T12:27:53.771812442Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236605398Z", + "ingested": "2022-05-17T12:27:53.771816632Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236611271Z", + "ingested": "2022-05-17T12:27:53.771820031Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236617045Z", + "ingested": "2022-05-17T12:27:53.771823167Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236621994Z", + "ingested": "2022-05-17T12:27:53.771826208Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T11:55:27.236627563Z", + "ingested": "2022-05-17T12:27:53.771830432Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", From 16579c1dec8f8bb08dad993140e861ab319c84f4 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 17 May 2022 15:19:45 +0200 Subject: [PATCH 07/15] Fix: support for vhost at the begining of the log --- .../test-access-basic.log-expected.json | 18 +++++++++--------- .../test-access-darwin.log-expected.json | 12 ++++++------ .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 +++++++++--------- .../test-access-vhost.log-expected.json | 6 +++++- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../test-error-basic.log-expected.json | 8 ++++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +++++++------- 10 files changed, 46 insertions(+), 42 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index c60a390015b..46ec6181d1b 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752676143Z", + "ingested": "2022-05-17T13:19:18.510251729Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752710961Z", + "ingested": "2022-05-17T13:19:18.510280294Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752720875Z", + "ingested": "2022-05-17T13:19:18.510288600Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752728496Z", + "ingested": "2022-05-17T13:19:18.510294676Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752735179Z", + "ingested": "2022-05-17T13:19:18.510300396Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752741297Z", + "ingested": "2022-05-17T13:19:18.510306110Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752747993Z", + "ingested": "2022-05-17T13:19:18.510311146Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752753866Z", + "ingested": "2022-05-17T13:19:18.510316033Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:52.752759288Z", + "ingested": "2022-05-17T13:19:18.510321002Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 06c8479b721..a6affd50c55 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131140454Z", + "ingested": "2022-05-17T13:19:18.854301744Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131166030Z", + "ingested": "2022-05-17T13:19:18.854324842Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131173502Z", + "ingested": "2022-05-17T13:19:18.854331590Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131179495Z", + "ingested": "2022-05-17T13:19:18.854337279Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131184861Z", + "ingested": "2022-05-17T13:19:18.854342361Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.131190159Z", + "ingested": "2022-05-17T13:19:18.854347132Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 782f00924b7..60eec1f9b71 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.199618007Z", + "ingested": "2022-05-17T13:19:18.924689457Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.199639521Z", + "ingested": "2022-05-17T13:19:18.924712071Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 13c741964d6..ba26b1755ac 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261379275Z", + "ingested": "2022-05-17T13:19:18.983488921Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261402676Z", + "ingested": "2022-05-17T13:19:18.983509400Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261409439Z", + "ingested": "2022-05-17T13:19:18.983515227Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261414730Z", + "ingested": "2022-05-17T13:19:18.983519894Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261419475Z", + "ingested": "2022-05-17T13:19:18.983524220Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261424008Z", + "ingested": "2022-05-17T13:19:18.983528303Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261428598Z", + "ingested": "2022-05-17T13:19:18.983532165Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261453917Z", + "ingested": "2022-05-17T13:19:18.983535943Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.261494716Z", + "ingested": "2022-05-17T13:19:18.983542089Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index e7cd3d107f5..b2dc1e9a96c 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -9,13 +9,16 @@ ] } }, + "destination": { + "domain": "vhost1.domaine.fr" + }, "ecs": { "version": "1.12.0" }, "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T12:27:53.353021263Z", + "ingested": "2022-05-17T13:19:19.071762287Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -44,6 +47,7 @@ "preserve_original_event" ], "url": { + "domain": "vhost1.domaine.fr", "original": "/hello", "path": "/hello" }, diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index d06b065ca9d..0e745b29287 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -16,7 +16,7 @@ processors: - grok: field: event.original patterns: - - '%{ADDRESS_LIST:apache.access.remote_ip_list} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '(%{HOSTNAME:destination.domain} )?%{ADDRESS_LIST:apache.access.remote_ip_list} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 25526d797c7..da91913298f 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.604423694Z", + "ingested": "2022-05-17T13:19:19.388103716Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.604440164Z", + "ingested": "2022-05-17T13:19:19.388158161Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.604444992Z", + "ingested": "2022-05-17T13:19:19.388168471Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.604448848Z", + "ingested": "2022-05-17T13:19:19.388174318Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 598487159db..057ac27ebd6 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.670123707Z", + "ingested": "2022-05-17T13:19:19.452723734Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.670141748Z", + "ingested": "2022-05-17T13:19:19.452742125Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index f8939240a46..4ac018f7e75 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.722420070Z", + "ingested": "2022-05-17T13:19:19.503250115Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index c3f5cb43fe5..836dedfb43a 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771795455Z", + "ingested": "2022-05-17T13:19:19.551609870Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771812442Z", + "ingested": "2022-05-17T13:19:19.551627952Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771816632Z", + "ingested": "2022-05-17T13:19:19.551633280Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771820031Z", + "ingested": "2022-05-17T13:19:19.551637663Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771823167Z", + "ingested": "2022-05-17T13:19:19.551641667Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771826208Z", + "ingested": "2022-05-17T13:19:19.551645434Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T12:27:53.771830432Z", + "ingested": "2022-05-17T13:19:19.551649159Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", From 96f1e32734d64c6c207a4a1405990d4189ae7c64 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 17 May 2022 23:09:49 +0200 Subject: [PATCH 08/15] Update: refactor try/catch, ctx can't be null, remove redundancy --- .../test-access-basic.log-expected.json | 18 +++++++-------- .../test-access-darwin.log-expected.json | 12 +++++----- .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 +++++++-------- .../test-access-vhost.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 23 ++++++++++--------- .../test-error-basic.log-expected.json | 8 +++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +++++------ 10 files changed, 53 insertions(+), 52 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 46ec6181d1b..a83230bcf7f 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510251729Z", + "ingested": "2022-05-17T21:07:02.024351124Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -66,7 +66,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510280294Z", + "ingested": "2022-05-17T21:07:02.024379364Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -130,7 +130,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510288600Z", + "ingested": "2022-05-17T21:07:02.024388899Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -169,7 +169,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510294676Z", + "ingested": "2022-05-17T21:07:02.024395895Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -229,7 +229,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510300396Z", + "ingested": "2022-05-17T21:07:02.024401676Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -290,7 +290,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510306110Z", + "ingested": "2022-05-17T21:07:02.024407088Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -338,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510311146Z", + "ingested": "2022-05-17T21:07:02.024412363Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -402,7 +402,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510316033Z", + "ingested": "2022-05-17T21:07:02.024417500Z", "kind": "event", "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -487,7 +487,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.510321002Z", + "ingested": "2022-05-17T21:07:02.024422479Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index a6affd50c55..0b29d768ed5 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854301744Z", + "ingested": "2022-05-17T21:07:02.382814534Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -65,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854324842Z", + "ingested": "2022-05-17T21:07:02.382834614Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -116,7 +116,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854331590Z", + "ingested": "2022-05-17T21:07:02.382841130Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -155,7 +155,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854337279Z", + "ingested": "2022-05-17T21:07:02.382846301Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -223,7 +223,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854342361Z", + "ingested": "2022-05-17T21:07:02.382851128Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -291,7 +291,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.854347132Z", + "ingested": "2022-05-17T21:07:02.382855677Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 60eec1f9b71..794e0221693 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -16,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.924689457Z", + "ingested": "2022-05-17T21:07:02.449930419Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -69,7 +69,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.924712071Z", + "ingested": "2022-05-17T21:07:02.449952976Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index ba26b1755ac..d8a5469bc02 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983488921Z", + "ingested": "2022-05-17T21:07:02.507798448Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -77,7 +77,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983509400Z", + "ingested": "2022-05-17T21:07:02.507821426Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -141,7 +141,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983515227Z", + "ingested": "2022-05-17T21:07:02.507828266Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -206,7 +206,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983519894Z", + "ingested": "2022-05-17T21:07:02.507833584Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -270,7 +270,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983524220Z", + "ingested": "2022-05-17T21:07:02.507838524Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -335,7 +335,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983528303Z", + "ingested": "2022-05-17T21:07:02.507844339Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -400,7 +400,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983532165Z", + "ingested": "2022-05-17T21:07:02.507849450Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -464,7 +464,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983535943Z", + "ingested": "2022-05-17T21:07:02.507854174Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -528,7 +528,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:18.983542089Z", + "ingested": "2022-05-17T21:07:02.507858905Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index b2dc1e9a96c..9c1acd93b28 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -18,7 +18,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T13:19:19.071762287Z", + "ingested": "2022-05-17T21:07:02.594130600Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 0e745b29287..e33d02fb84e 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -47,18 +47,19 @@ processors: CIDR class_b_network = new CIDR('172.16.0.0/12'); CIDR class_c_network = new CIDR('192.168.0.0/16'); - return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip); + try { + return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip); + } catch (IllegalArgumentException e) { + return false; + } } try { - if (ctx?.source == null){ + if (ctx.source == null) { Map map = new HashMap(); ctx.put("source", map); } - ctx.source.address = null; - if (ctx.apache.access.remote_ip_list == null) { - return; - } + def found = false; for (def item : ctx.apache.access.remote_ip_list) { if (!isPrivateCIDR(item)) { @@ -94,11 +95,11 @@ processors: - set: field: event.outcome value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399" - grok: field: source.address ignore_missing: true @@ -150,10 +151,10 @@ processors: - set: field: tls.cipher value: '{{apache.access.ssl.cipher}}' - if: ctx?.apache?.access?.ssl?.cipher != null + if: ctx.apache?.access?.ssl?.cipher != null - script: lang: painless - if: ctx?.apache?.access?.ssl?.protocol != null + if: ctx.apache?.access?.ssl?.protocol != null source: >- def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); if (parts.length != 2) { @@ -191,7 +192,7 @@ processors: handleMap(ctx); - remove: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true on_failure: diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index da91913298f..0a08bf4d23c 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.388103716Z", + "ingested": "2022-05-17T21:07:02.826505587Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.388158161Z", + "ingested": "2022-05-17T21:07:02.826523242Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.388168471Z", + "ingested": "2022-05-17T21:07:02.826528266Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.388174318Z", + "ingested": "2022-05-17T21:07:02.826532100Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 057ac27ebd6..8c6292ce3ba 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.452723734Z", + "ingested": "2022-05-17T21:07:02.884601564Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.452742125Z", + "ingested": "2022-05-17T21:07:02.884617860Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 4ac018f7e75..bd40dbfdad8 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.503250115Z", + "ingested": "2022-05-17T21:07:02.931968981Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 836dedfb43a..7fbc04f5566 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551609870Z", + "ingested": "2022-05-17T21:07:02.978880092Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551627952Z", + "ingested": "2022-05-17T21:07:02.978897201Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551633280Z", + "ingested": "2022-05-17T21:07:02.978902267Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551637663Z", + "ingested": "2022-05-17T21:07:02.978906246Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551641667Z", + "ingested": "2022-05-17T21:07:02.978909937Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551645434Z", + "ingested": "2022-05-17T21:07:02.978913292Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T13:19:19.551649159Z", + "ingested": "2022-05-17T21:07:02.978916574Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", From ac90936a99b6546b266d5ad8cf49c179a4998b90 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Wed, 18 May 2022 09:37:29 +0200 Subject: [PATCH 09/15] Update: Updated format for apache logs apache.access.remote_adresses contain all IP/HOST the request rebounded of X-Forwarded-For is optional, should be tacked a the end of the logs network.forwarded_ip exist only when X-Forwarded_for is found Merge 2 similar pattern in GROK processor --- packages/apache/_dev/deploy/docker/httpd.conf | 4 +- .../_dev/test/pipeline/test-access-basic.log | 7 +- .../test-access-basic.log-expected.json | 133 ++++++++++++------ .../test-access-darwin.log-expected.json | 42 ++---- .../test-access-ssl-request.log-expected.json | 16 +-- .../test-access-ubuntu.log-expected.json | 63 +++------ .../test-access-vhost.log-expected.json | 7 +- .../elasticsearch/ingest_pipeline/default.yml | 32 ++--- .../data_stream/access/fields/fields.yml | 6 +- .../test-error-basic.log-expected.json | 8 +- .../test-error-darwin.log-expected.json | 4 +- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 +- packages/apache/docs/README.md | 2 +- 14 files changed, 173 insertions(+), 167 deletions(-) diff --git a/packages/apache/_dev/deploy/docker/httpd.conf b/packages/apache/_dev/deploy/docker/httpd.conf index 7db7bfc70d1..fad5c9924e4 100644 --- a/packages/apache/_dev/deploy/docker/httpd.conf +++ b/packages/apache/_dev/deploy/docker/httpd.conf @@ -281,7 +281,7 @@ LogLevel warn # The following directives define some format nicknames for use with # a CustomLog directive (see below). # - LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common @@ -296,7 +296,7 @@ LogLevel warn # define per- access logfiles, transactions will be # logged therein and *not* in this file. # - CustomLog "/usr/local/apache2/logs/access.log" common + CustomLog "/usr/local/apache2/logs/access.log" combined # # If you prefer a logfile with access, agent, and referer information diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log index d3e6b494280..6f995c73596 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -4,6 +4,7 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" -monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" -10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" -2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="-" +89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.0.0.2,10.0.0.1" +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.225.192.17, 10.2.2.121" +monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2" \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index a83230bcf7f..f9c6eb82c25 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -4,7 +4,7 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "::1" ] } @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024351124Z", + "ingested": "2022-05-18T07:36:27.553478080Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -32,9 +32,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "::1" - }, "source": { "address": "::1", "ip": "::1" @@ -55,7 +52,7 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -66,7 +63,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024379364Z", + "ingested": "2022-05-18T07:36:27.553479853Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -84,9 +81,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -119,7 +113,7 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "::1" ] } @@ -130,7 +124,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024388899Z", + "ingested": "2022-05-18T07:36:27.553480378Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -140,9 +134,6 @@ "status_code": 408 } }, - "network": { - "forwarded_ip": "::1" - }, "source": { "address": "::1", "ip": "::1" @@ -158,7 +149,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "172.17.0.1" ] } @@ -169,7 +160,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024395895Z", + "ingested": "2022-05-18T07:36:27.553480820Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -187,9 +178,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "172.17.0.1" - }, "source": { "address": "172.17.0.1", "ip": "172.17.0.1" @@ -221,7 +209,11 @@ { "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "monitoring-server" + ] + } }, "ecs": { "version": "1.12.0" @@ -229,7 +221,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024401676Z", + "ingested": "2022-05-18T07:36:27.553481267Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -279,7 +271,7 @@ "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "127.0.0.1" ] } @@ -290,7 +282,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024407088Z", + "ingested": "2022-05-18T07:36:27.553481680Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -306,9 +298,6 @@ "status_code": 408 } }, - "network": { - "forwarded_ip": "127.0.0.1" - }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" @@ -330,7 +319,11 @@ { "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "monitoring-server" + ] + } }, "ecs": { "version": "1.12.0" @@ -338,9 +331,9 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024412363Z", + "ingested": "2022-05-18T07:36:27.553482097Z", "kind": "event", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", "outcome": "success" }, "http": { @@ -389,7 +382,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "10.0.0.2", "10.0.0.1", "89.160.20.112" @@ -402,9 +395,9 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024417500Z", + "ingested": "2022-05-18T07:36:27.553482512Z", "kind": "event", - "original": "10.0.0.2, 10.0.0.1, 89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" }, "http": { @@ -421,7 +414,7 @@ "version": "1.1" }, "network": { - "forwarded_ip": "89.160.20.112" + "forwarded_ip": "10.0.0.2" }, "source": { "address": "89.160.20.112", @@ -474,10 +467,10 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "remote_ip_list": [ - "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "remote_addresses": [ "10.225.192.17", - "10.2.2.121" + "10.2.2.121", + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" ] } }, @@ -487,9 +480,9 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.024422479Z", + "ingested": "2022-05-18T07:36:27.553482931Z", "kind": "event", - "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6, 10.225.192.17, 10.2.2.121 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", "outcome": "success" }, "http": { @@ -506,7 +499,7 @@ "version": "1.1" }, "network": { - "forwarded_ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + "forwarded_ip": "10.225.192.17" }, "source": { "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", @@ -545,6 +538,66 @@ }, "version": "15.0.a2" } + }, + { + "@timestamp": "2022-05-17T21:41:43.000Z", + "apache": { + "access": { + "remote_addresses": [ + "192.168.0.2", + "monitoring-server" + ] + } + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-05-18T07:36:27.553483352Z", + "kind": "event", + "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "192.168.0.2" + }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "curl", + "original": "curl/7.79.1", + "version": "7.79.1" + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 0b29d768ed5..f0269210e63 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -4,7 +4,7 @@ "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "::1" ] } @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382814534Z", + "ingested": "2022-05-18T07:36:27.629478592Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -32,9 +32,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "::1" - }, "source": { "address": "::1", "ip": "::1" @@ -54,7 +51,7 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "::1" ] } @@ -65,7 +62,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382834614Z", + "ingested": "2022-05-18T07:36:27.629480611Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -82,9 +79,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "::1" - }, "source": { "address": "::1", "ip": "::1" @@ -105,7 +99,7 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "::1" ] } @@ -116,7 +110,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382841130Z", + "ingested": "2022-05-18T07:36:27.629481055Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -126,9 +120,6 @@ "status_code": 408 } }, - "network": { - "forwarded_ip": "::1" - }, "source": { "address": "::1", "ip": "::1" @@ -144,7 +135,7 @@ "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "89.160.20.156" ] } @@ -155,7 +146,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382846301Z", + "ingested": "2022-05-18T07:36:27.629481432Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -172,9 +163,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "89.160.20.156" - }, "source": { "address": "89.160.20.156", "as": { @@ -212,7 +200,7 @@ "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "89.160.20.156" ] } @@ -223,7 +211,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382851128Z", + "ingested": "2022-05-18T07:36:27.629481792Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -240,9 +228,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "89.160.20.156" - }, "source": { "address": "89.160.20.156", "as": { @@ -280,7 +265,7 @@ "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "89.160.20.156" ] } @@ -291,7 +276,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.382855677Z", + "ingested": "2022-05-18T07:36:27.629482162Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" @@ -308,9 +293,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "89.160.20.156" - }, "source": { "address": "89.160.20.156", "as": { diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 794e0221693..0220b10a27e 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -4,6 +4,9 @@ "@timestamp": "2018-08-10T07:45:56.000Z", "apache": { "access": { + "remote_addresses": [ + "172.30.0.119" + ], "ssl": { "cipher": "ECDHE-RSA-AES128-GCM-SHA256", "protocol": "TLSv1.2" @@ -16,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.449930419Z", + "ingested": "2022-05-18T07:36:27.680318105Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -31,9 +34,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "172.30.0.119" - }, "source": { "address": "172.30.0.119", "ip": "172.30.0.119" @@ -57,6 +57,9 @@ "@timestamp": "2019-10-16T09:53:47.000Z", "apache": { "access": { + "remote_addresses": [ + "89.160.20.156" + ], "ssl": { "cipher": "ECDHE-RSA-AES128-GCM-SHA256", "protocol": "TLSv1.2" @@ -69,7 +72,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.449952976Z", + "ingested": "2022-05-18T07:36:27.680320111Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, @@ -79,9 +82,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "89.160.20.156" - }, "source": { "address": "89.160.20.156", "as": { diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index d8a5469bc02..a1970ffedec 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -4,7 +4,7 @@ "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "127.0.0.1" ] } @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507798448Z", + "ingested": "2022-05-18T07:36:27.727176381Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -33,9 +33,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "127.0.0.1" - }, "source": { "address": "127.0.0.1", "ip": "127.0.0.1" @@ -66,7 +63,7 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -77,7 +74,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507821426Z", + "ingested": "2022-05-18T07:36:27.727178381Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -95,9 +92,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -130,7 +124,7 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -141,7 +135,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507828266Z", + "ingested": "2022-05-18T07:36:27.727178806Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -159,9 +153,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -195,7 +186,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -206,7 +197,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507833584Z", + "ingested": "2022-05-18T07:36:27.727179193Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -224,9 +215,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -259,7 +247,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -270,7 +258,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507838524Z", + "ingested": "2022-05-18T07:36:27.727179548Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -288,9 +276,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -324,7 +309,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -335,7 +320,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507844339Z", + "ingested": "2022-05-18T07:36:27.727179913Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -353,9 +338,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -389,7 +371,7 @@ "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -400,7 +382,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507849450Z", + "ingested": "2022-05-18T07:36:27.727180267Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -418,9 +400,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -453,7 +432,7 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -464,7 +443,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507854174Z", + "ingested": "2022-05-18T07:36:27.727180617Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -482,9 +461,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" @@ -517,7 +493,7 @@ "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.1" ] } @@ -528,7 +504,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.507858905Z", + "ingested": "2022-05-18T07:36:27.727180959Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -546,9 +522,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.1" - }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 9c1acd93b28..f6af5f537df 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -4,7 +4,7 @@ "@timestamp": "2016-12-26T16:22:14.000Z", "apache": { "access": { - "remote_ip_list": [ + "remote_addresses": [ "192.168.33.2" ] } @@ -18,7 +18,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-17T21:07:02.594130600Z", + "ingested": "2022-05-18T07:36:27.791821069Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -36,9 +36,6 @@ }, "version": "1.1" }, - "network": { - "forwarded_ip": "192.168.33.2" - }, "source": { "address": "192.168.33.2", "ip": "192.168.33.2" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index e33d02fb84e..3f76a7bb4bf 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -16,14 +16,10 @@ processors: - grok: field: event.original patterns: - - '(%{HOSTNAME:destination.domain} )?%{ADDRESS_LIST:apache.access.remote_ip_list} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" - %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?( X-Forwarded-For="%{ADDRESS_LIST:apache.access.remote_addresses}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} @@ -33,14 +29,22 @@ processors: pattern_definitions: ADDRESS_LIST: (%{IP})("?,?\s*(%{IP}))* - split: - field: apache.access.remote_ip_list - separator: '"?,?\s+' + field: apache.access.remote_addresses + separator: '"?,\s*' ignore_missing: true + - set: + field: network.forwarded_ip + value: "{{{apache.access.remote_addresses.0}}}" + if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 + - append: + field: apache.access.remote_addresses + value: ["{{source.address}}"] + if: ctx.source?.address != null - script: - if: ctx.apache?.access?.remote_ip_list != null && ctx.apache.access.remote_ip_list.length > 0 + if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 lang: painless tag: Get source address - description: Extract from remote_ip_list, the first non-private IP to source.address + description: Extract from remote_addresses, the first non-private IP to source.address source: >- boolean isPrivateCIDR(def ip) { CIDR class_a_network = new CIDR('10.0.0.0/8'); @@ -61,7 +65,7 @@ processors: } def found = false; - for (def item : ctx.apache.access.remote_ip_list) { + for (def item : ctx.apache.access.remote_addresses) { if (!isPrivateCIDR(item)) { ctx.source.address = item; found = true; @@ -69,7 +73,7 @@ processors: } } if (!found) { - ctx.source.address = ctx.apache.access.remote_ip_list[0]; + ctx.source.address = ctx.apache.access.remote_addresses[0]; } } catch (Exception e) { @@ -105,10 +109,6 @@ processors: ignore_missing: true patterns: - ^(%{IP:source.ip}|%{HOSTNAME:source.domain})$ - - set: - field: network.forwarded_ip - copy_from: source.ip - ignore_empty_value: true - remove: field: event.created ignore_missing: true diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index a2ba06392dd..f9dc5e7bd4e 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -10,7 +10,7 @@ description: | SSL cipher name. - name: nginx.access - - name: remote_ip_list - type: ip + - name: remote_addresses + type: keyword description: | - An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. + An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 0a08bf4d23c..2b2d4f726a5 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.826505587Z", + "ingested": "2022-05-18T07:36:27.909998098Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.826523242Z", + "ingested": "2022-05-18T07:36:27.910000464Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.826528266Z", + "ingested": "2022-05-18T07:36:27.910001023Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.826532100Z", + "ingested": "2022-05-18T07:36:27.910001461Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 8c6292ce3ba..64fbb3404f4 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.884601564Z", + "ingested": "2022-05-18T07:36:27.964148653Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.884617860Z", + "ingested": "2022-05-18T07:36:27.964150869Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index bd40dbfdad8..3e32fbfe276 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.931968981Z", + "ingested": "2022-05-18T07:36:28.011341031Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 7fbc04f5566..12eb11837be 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978880092Z", + "ingested": "2022-05-18T07:36:28.053016068Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978897201Z", + "ingested": "2022-05-18T07:36:28.053018558Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978902267Z", + "ingested": "2022-05-18T07:36:28.053019088Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978906246Z", + "ingested": "2022-05-18T07:36:28.053019481Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978909937Z", + "ingested": "2022-05-18T07:36:28.053019856Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978913292Z", + "ingested": "2022-05-18T07:36:28.053020222Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-17T21:07:02.978916574Z", + "ingested": "2022-05-18T07:36:28.053020590Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index cece7f9753c..338caeeda39 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,7 +19,7 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.remote_ip_list | An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. Real source IP is restored to `source.ip`. | ip | +| apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | From 3e5c9f8e4a72a8f8430205a0dfe66a5eacd79850 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Mon, 23 May 2022 07:11:19 +0200 Subject: [PATCH 10/15] Update: Fill client.ip instead of source.address --- .../test-access-basic.log-expected.json | 29 ++++++++++++------- .../test-access-darwin.log-expected.json | 12 ++++---- .../test-access-ssl-request.log-expected.json | 4 +-- .../test-access-ubuntu.log-expected.json | 18 ++++++------ .../test-access-vhost.log-expected.json | 2 +- .../elasticsearch/ingest_pipeline/default.yml | 20 ++++++------- .../apache/data_stream/access/fields/ecs.yml | 2 ++ .../test-error-basic.log-expected.json | 8 ++--- .../test-error-darwin.log-expected.json | 4 +-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 ++++----- packages/apache/docs/README.md | 1 + 12 files changed, 64 insertions(+), 52 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index f9c6eb82c25..db2f336560c 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553478080Z", + "ingested": "2022-05-23T05:10:26.971951941Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -63,7 +63,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553479853Z", + "ingested": "2022-05-23T05:10:26.972008274Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -124,7 +124,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553480378Z", + "ingested": "2022-05-23T05:10:26.972013140Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -160,7 +160,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553480820Z", + "ingested": "2022-05-23T05:10:26.972015295Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -221,7 +221,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553481267Z", + "ingested": "2022-05-23T05:10:26.972017278Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -282,7 +282,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553481680Z", + "ingested": "2022-05-23T05:10:26.972019184Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -331,7 +331,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553482097Z", + "ingested": "2022-05-23T05:10:26.972021241Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", "outcome": "success" @@ -389,13 +389,16 @@ ] } }, + "client": { + "ip": "10.0.0.2" + }, "ecs": { "version": "1.12.0" }, "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553482512Z", + "ingested": "2022-05-23T05:10:26.972023105Z", "kind": "event", "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" @@ -474,13 +477,16 @@ ] } }, + "client": { + "ip": "10.225.192.17" + }, "ecs": { "version": "1.12.0" }, "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553482931Z", + "ingested": "2022-05-23T05:10:26.972025031Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", "outcome": "success" @@ -549,13 +555,16 @@ ] } }, + "client": { + "ip": "192.168.0.2" + }, "ecs": { "version": "1.12.0" }, "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.553483352Z", + "ingested": "2022-05-23T05:10:26.972026850Z", "kind": "event", "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index f0269210e63..8614fa2fada 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629478592Z", + "ingested": "2022-05-23T05:10:27.045293780Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -62,7 +62,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629480611Z", + "ingested": "2022-05-23T05:10:27.045342606Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -110,7 +110,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629481055Z", + "ingested": "2022-05-23T05:10:27.045345128Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -146,7 +146,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629481432Z", + "ingested": "2022-05-23T05:10:27.045346777Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -211,7 +211,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629481792Z", + "ingested": "2022-05-23T05:10:27.045348417Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -276,7 +276,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.629482162Z", + "ingested": "2022-05-23T05:10:27.045349986Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 0220b10a27e..85b82537197 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -19,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.680318105Z", + "ingested": "2022-05-23T05:10:27.097836567Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -72,7 +72,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.680320111Z", + "ingested": "2022-05-23T05:10:27.097843491Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index a1970ffedec..9c33c8d9ec9 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727176381Z", + "ingested": "2022-05-23T05:10:27.141843114Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -74,7 +74,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727178381Z", + "ingested": "2022-05-23T05:10:27.141849816Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -135,7 +135,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727178806Z", + "ingested": "2022-05-23T05:10:27.141851670Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -197,7 +197,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727179193Z", + "ingested": "2022-05-23T05:10:27.141853310Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -258,7 +258,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727179548Z", + "ingested": "2022-05-23T05:10:27.141854838Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -320,7 +320,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727179913Z", + "ingested": "2022-05-23T05:10:27.141856348Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -382,7 +382,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727180267Z", + "ingested": "2022-05-23T05:10:27.141857878Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -443,7 +443,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727180617Z", + "ingested": "2022-05-23T05:10:27.141859349Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -504,7 +504,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.727180959Z", + "ingested": "2022-05-23T05:10:27.141860879Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index f6af5f537df..3854db39c75 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -18,7 +18,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-18T07:36:27.791821069Z", + "ingested": "2022-05-23T05:10:27.209455786Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 3f76a7bb4bf..e9c332c308b 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -36,15 +36,11 @@ processors: field: network.forwarded_ip value: "{{{apache.access.remote_addresses.0}}}" if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 - - append: - field: apache.access.remote_addresses - value: ["{{source.address}}"] - if: ctx.source?.address != null - script: if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 lang: painless tag: Get source address - description: Extract from remote_addresses, the first non-private IP to source.address + description: Extract from remote_addresses, the first non-private IP to ctx.client.ip source: >- boolean isPrivateCIDR(def ip) { CIDR class_a_network = new CIDR('10.0.0.0/8'); @@ -59,26 +55,30 @@ processors: } try { - if (ctx.source == null) { + if (ctx.client == null) { Map map = new HashMap(); - ctx.put("source", map); + ctx.put("client", map); } def found = false; for (def item : ctx.apache.access.remote_addresses) { if (!isPrivateCIDR(item)) { - ctx.source.address = item; + ctx.client.ip = item; found = true; break; } } if (!found) { - ctx.source.address = ctx.apache.access.remote_addresses[0]; + ctx.client.ip = ctx.apache.access.remote_addresses[0]; } } catch (Exception e) { - ctx.source.address = null; + ctx.client.ip = null; } + - append: + field: apache.access.remote_addresses + value: ["{{source.address}}"] + if: ctx.source?.address != null - uri_parts: field: _tmp.url_orig ignore_failure: true diff --git a/packages/apache/data_stream/access/fields/ecs.yml b/packages/apache/data_stream/access/fields/ecs.yml index 7488d3b683a..e9c1f6c7d85 100644 --- a/packages/apache/data_stream/access/fields/ecs.yml +++ b/packages/apache/data_stream/access/fields/ecs.yml @@ -1,3 +1,5 @@ +- external: ecs + name: client.ip - external: ecs name: destination.domain - external: ecs diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index 2b2d4f726a5..dc46cb261ce 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.909998098Z", + "ingested": "2022-05-23T05:10:27.341676182Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.910000464Z", + "ingested": "2022-05-23T05:10:27.341682693Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.910001023Z", + "ingested": "2022-05-23T05:10:27.341722105Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.910001461Z", + "ingested": "2022-05-23T05:10:27.341732343Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 64fbb3404f4..e377c678ffd 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.964148653Z", + "ingested": "2022-05-23T05:10:27.387066444Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:27.964150869Z", + "ingested": "2022-05-23T05:10:27.387073231Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 3e32fbfe276..e99cc6c1865 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.011341031Z", + "ingested": "2022-05-23T05:10:27.427713736Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 12eb11837be..bbb903af0f0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053016068Z", + "ingested": "2022-05-23T05:10:27.468313715Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053018558Z", + "ingested": "2022-05-23T05:10:27.468320043Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053019088Z", + "ingested": "2022-05-23T05:10:27.468321902Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053019481Z", + "ingested": "2022-05-23T05:10:27.468323535Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053019856Z", + "ingested": "2022-05-23T05:10:27.468325105Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053020222Z", + "ingested": "2022-05-23T05:10:27.468326653Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-18T07:36:28.053020590Z", + "ingested": "2022-05-23T05:10:27.468328217Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index 338caeeda39..f1de6d940da 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -22,6 +22,7 @@ Access logs collects the Apache access logs. | apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | +| client.ip | IP address of the client (IPv4 or IPv6). | ip | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | From 73ecf8372357982b4c9a1980474e272231c52908 Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Fri, 3 Jun 2022 12:19:13 +0200 Subject: [PATCH 11/15] Update: Add some documentation to explain supported format --- packages/apache/_dev/build/docs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/apache/_dev/build/docs/README.md b/packages/apache/_dev/build/docs/README.md index 932b335cd78..9cb6c6e973d 100644 --- a/packages/apache/_dev/build/docs/README.md +++ b/packages/apache/_dev/build/docs/README.md @@ -16,6 +16,24 @@ Access logs collects the Apache access logs. {{fields "access"}} +Supported format for the access logs are: + +- [Common Log Format](https://en.wikipedia.org/wiki/Common_Log_Format) + - Defined in apache `LogFormat` by : + >```%h %l %u %t \"%r\" %>s %b``` + - Example: + > `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326` +- Combined Log Format + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` +- Combined Log Format + x_forwarded_for header + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)" X-Forwarded-For="10.225.192.17, 10.2.2.121"``` + ### Error Logs Error logs collects the Apache error logs. From 15d06142c45de604bb61877c34a7e91cebf9b0db Mon Sep 17 00:00:00 2001 From: Paulo Date: Tue, 7 Jun 2022 08:25:40 +0200 Subject: [PATCH 12/15] Update packages/apache/_dev/build/docs/README.md Co-authored-by: Andrew Kroh --- packages/apache/_dev/build/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apache/_dev/build/docs/README.md b/packages/apache/_dev/build/docs/README.md index 9cb6c6e973d..7ab6354742d 100644 --- a/packages/apache/_dev/build/docs/README.md +++ b/packages/apache/_dev/build/docs/README.md @@ -28,7 +28,7 @@ Supported format for the access logs are: >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` - Example: >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` -- Combined Log Format + x_forwarded_for header +- Combined Log Format + X-Forwarded-For header - Defined in apache `LogFormat` by: >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` - Example: From 1047981e396146ee223fdc7d4d96d31e3f43196a Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 7 Jun 2022 08:26:12 +0200 Subject: [PATCH 13/15] Update: Build integration to generate Md doc --- packages/apache/docs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index f1de6d940da..4589e1d116d 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -117,6 +117,24 @@ Access logs collects the Apache access logs. | user_agent.version | Version of the user agent. | keyword | +Supported format for the access logs are: + +- [Common Log Format](https://en.wikipedia.org/wiki/Common_Log_Format) + - Defined in apache `LogFormat` by : + >```%h %l %u %t \"%r\" %>s %b``` + - Example: + > `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326` +- Combined Log Format + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` +- Combined Log Format + x_forwarded_for header + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)" X-Forwarded-For="10.225.192.17, 10.2.2.121"``` + ### Error Logs Error logs collects the Apache error logs. From 499515f2dea2aa670a276eefc402d662098e4dcc Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Tue, 7 Jun 2022 08:53:46 +0200 Subject: [PATCH 14/15] Update: manifest.yml --- packages/apache/docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index 4589e1d116d..8d1f488eb75 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -129,7 +129,7 @@ Supported format for the access logs are: >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` - Example: >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` -- Combined Log Format + x_forwarded_for header +- Combined Log Format + X-Forwarded-For header - Defined in apache `LogFormat` by: >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` - Example: From 2c91c7831ca2ef30cfcd7c1d2514b0e9e3721b3e Mon Sep 17 00:00:00 2001 From: Paul Le Tilly Date: Fri, 29 Jul 2022 11:30:08 +0200 Subject: [PATCH 15/15] Fix: issue during merge --- packages/apache/changelog.yml | 1 + .../test-access-basic.log-expected.json | 20 +++++++++---------- .../test-access-darwin.log-expected.json | 12 +++++------ .../test-access-ssl-request.log-expected.json | 4 ++-- .../test-access-ubuntu.log-expected.json | 18 ++++++++--------- .../test-access-vhost.log-expected.json | 2 +- .../test-error-basic.log-expected.json | 8 ++++---- .../test-error-darwin.log-expected.json | 4 ++-- .../test-error-trace.log-expected.json | 2 +- .../test-error-ubuntu.log-expected.json | 14 ++++++------- packages/apache/manifest.yml | 2 +- 11 files changed, 44 insertions(+), 43 deletions(-) diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index 496504c3025..2ecf68f6047 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -5,6 +5,7 @@ type: enhancement link: https://github.com/elastic/integrations/pull/3251 - version: "1.5.0" + changes: - description: Use new labels for source license and subscription type: enhancement link: https://github.com/elastic/integrations/issues/3816 diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index db2f336560c..345ea54c4b9 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.971951941Z", + "ingested": "2022-07-29T12:32:45.200116114Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -63,7 +63,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972008274Z", + "ingested": "2022-07-29T12:32:45.200149914Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -124,7 +124,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972013140Z", + "ingested": "2022-07-29T12:32:45.200158562Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -160,7 +160,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972015295Z", + "ingested": "2022-07-29T12:32:45.200165012Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -221,7 +221,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972017278Z", + "ingested": "2022-07-29T12:32:45.200170453Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -282,7 +282,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972019184Z", + "ingested": "2022-07-29T12:32:45.200175624Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -331,7 +331,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972021241Z", + "ingested": "2022-07-29T12:32:45.200180593Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", "outcome": "success" @@ -398,7 +398,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972023105Z", + "ingested": "2022-07-29T12:32:45.200185417Z", "kind": "event", "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" @@ -486,7 +486,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972025031Z", + "ingested": "2022-07-29T12:32:45.200190094Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", "outcome": "success" @@ -564,7 +564,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:26.972026850Z", + "ingested": "2022-07-29T12:32:45.200194779Z", "kind": "event", "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 8614fa2fada..e87e7538905 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045293780Z", + "ingested": "2022-07-29T12:32:45.405343801Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -62,7 +62,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045342606Z", + "ingested": "2022-07-29T12:32:45.405359749Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -110,7 +110,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045345128Z", + "ingested": "2022-07-29T12:32:45.405365252Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -146,7 +146,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045346777Z", + "ingested": "2022-07-29T12:32:45.405369669Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -211,7 +211,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045348417Z", + "ingested": "2022-07-29T12:32:45.405373693Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -276,7 +276,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.045349986Z", + "ingested": "2022-07-29T12:32:45.405377632Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 85b82537197..7aa0164b5c2 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -19,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.097836567Z", + "ingested": "2022-07-29T12:32:45.472518977Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -72,7 +72,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.097843491Z", + "ingested": "2022-07-29T12:32:45.472546575Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 9c33c8d9ec9..122d1046537 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -15,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141843114Z", + "ingested": "2022-07-29T12:32:45.531606809Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -74,7 +74,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141849816Z", + "ingested": "2022-07-29T12:32:45.531626408Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -135,7 +135,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141851670Z", + "ingested": "2022-07-29T12:32:45.531632562Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -197,7 +197,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141853310Z", + "ingested": "2022-07-29T12:32:45.531637660Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -258,7 +258,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141854838Z", + "ingested": "2022-07-29T12:32:45.531642297Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -320,7 +320,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141856348Z", + "ingested": "2022-07-29T12:32:45.531646911Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -382,7 +382,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141857878Z", + "ingested": "2022-07-29T12:32:45.531651400Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -443,7 +443,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141859349Z", + "ingested": "2022-07-29T12:32:45.531655775Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -504,7 +504,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.141860879Z", + "ingested": "2022-07-29T12:32:45.531660022Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 3854db39c75..d5d7cf93938 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -18,7 +18,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-05-23T05:10:27.209455786Z", + "ingested": "2022-07-29T12:32:45.625025669Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index dc46cb261ce..adb241ff953 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.341676182Z", + "ingested": "2022-07-29T12:32:45.886133788Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -43,7 +43,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.341682693Z", + "ingested": "2022-07-29T12:32:45.886154200Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", @@ -72,7 +72,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.341722105Z", + "ingested": "2022-07-29T12:32:45.886161220Z", "kind": "event", "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "timezone": "GMT+2", @@ -129,7 +129,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.341732343Z", + "ingested": "2022-07-29T12:32:45.886167076Z", "kind": "event", "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index e377c678ffd..9c743f595d0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.387066444Z", + "ingested": "2022-07-29T12:32:45.957125294Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", @@ -41,7 +41,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.387073231Z", + "ingested": "2022-07-29T12:32:45.957148462Z", "kind": "event", "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index e99cc6c1865..3c739afc052 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -12,7 +12,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.427713736Z", + "ingested": "2022-07-29T12:32:46.015505043Z", "kind": "event", "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index bbb903af0f0..2a3ff1117db 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -10,7 +10,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468313715Z", + "ingested": "2022-07-29T12:32:46.067824384Z", "kind": "event", "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", @@ -34,7 +34,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468320043Z", + "ingested": "2022-07-29T12:32:46.067850374Z", "kind": "event", "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "timezone": "GMT+2", @@ -70,7 +70,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468321902Z", + "ingested": "2022-07-29T12:32:46.067856458Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -101,7 +101,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468323535Z", + "ingested": "2022-07-29T12:32:46.067861616Z", "kind": "event", "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", @@ -132,7 +132,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468325105Z", + "ingested": "2022-07-29T12:32:46.067866350Z", "kind": "event", "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "timezone": "GMT+2", @@ -163,7 +163,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468326653Z", + "ingested": "2022-07-29T12:32:46.067870964Z", "kind": "event", "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "timezone": "GMT+2", @@ -194,7 +194,7 @@ }, "event": { "category": "web", - "ingested": "2022-05-23T05:10:27.468328217Z", + "ingested": "2022-07-29T12:32:46.067875366Z", "kind": "event", "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "timezone": "GMT+2", diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index cf59d4d75d8..32e87ccb30a 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: apache title: Apache HTTP Server -version: 1.5.0 +version: 1.6.0 license: basic source: license: Elastic-2.0