From 176203b234e1e5f552b70500f855648a62b9147d Mon Sep 17 00:00:00 2001 From: Paulo Date: Mon, 29 Aug 2022 08:24:24 +0200 Subject: [PATCH] Update: Add support for x_forwarded_for headers in apaches access logs (#3251) * Update: Add support for x_forwarded_for headers in apaches access logs This Pr is heavily inpired by the work done in [this PR](https://github.com/elastic/beats/pull/4417) It is adressing [this ER](https://github.com/elastic/enhancements/issues/14402). `Grok` pattern has been updated to match logs starting with a list of IP adresses and store all those ip in `apache.access.remote_ip`. This pattern is heavily insipred by the one in the nginx integration. I also decided to fill a new field `network.forwarded_ip` as it seems to be the perfect fit. Co-authored-by: Andrew Kroh --- packages/apache/_dev/build/docs/README.md | 18 + packages/apache/_dev/deploy/docker/httpd.conf | 4 +- packages/apache/changelog.yml | 5 + .../_dev/test/pipeline/test-access-basic.log | 5 +- .../test-access-basic.log-expected.json | 528 +++++++++++++----- .../test-access-darwin.log-expected.json | 322 ++++++----- .../test-access-ssl-request.log-expected.json | 104 ++-- .../test-access-ubuntu.log-expected.json | 524 +++++++++-------- .../test-access-vhost.log-expected.json | 63 ++- .../elasticsearch/ingest_pipeline/default.yml | 71 ++- .../apache/data_stream/access/fields/ecs.yml | 4 + .../data_stream/access/fields/fields.yml | 5 + .../test-error-basic.log-expected.json | 168 +++--- .../test-error-darwin.log-expected.json | 44 +- .../test-error-trace.log-expected.json | 28 +- .../test-error-ubuntu.log-expected.json | 186 +++--- packages/apache/docs/README.md | 23 +- packages/apache/manifest.yml | 2 +- 18 files changed, 1269 insertions(+), 835 deletions(-) diff --git a/packages/apache/_dev/build/docs/README.md b/packages/apache/_dev/build/docs/README.md index 932b335cd78..7ab6354742d 100644 --- a/packages/apache/_dev/build/docs/README.md +++ b/packages/apache/_dev/build/docs/README.md @@ -16,6 +16,24 @@ Access logs collects the Apache access logs. {{fields "access"}} +Supported format for the access logs are: + +- [Common Log Format](https://en.wikipedia.org/wiki/Common_Log_Format) + - Defined in apache `LogFormat` by : + >```%h %l %u %t \"%r\" %>s %b``` + - Example: + > `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326` +- Combined Log Format + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` +- Combined Log Format + X-Forwarded-For header + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)" X-Forwarded-For="10.225.192.17, 10.2.2.121"``` + ### Error Logs Error logs collects the Apache error logs. diff --git a/packages/apache/_dev/deploy/docker/httpd.conf b/packages/apache/_dev/deploy/docker/httpd.conf index f402947317d..fad5c9924e4 100644 --- a/packages/apache/_dev/deploy/docker/httpd.conf +++ b/packages/apache/_dev/deploy/docker/httpd.conf @@ -281,7 +281,7 @@ LogLevel warn # The following directives define some format nicknames for use with # a CustomLog directive (see below). # - LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common @@ -296,7 +296,7 @@ LogLevel warn # define per- access logfiles, transactions will be # logged therein and *not* in this file. # - CustomLog "/usr/local/apache2/logs/access.log" common + CustomLog "/usr/local/apache2/logs/access.log" combined # # If you prefer a logfile with access, agent, and referer information diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index c1635730bb0..52df250b676 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.6.0" + changes: + - description: Add support for x_forwarded_for header + type: enhancement + link: https://github.com/elastic/integrations/pull/3251 - version: "1.5.1" changes: - description: Remove unused visualizations diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log index 4e2cbbe7e54..6f995c73596 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -4,4 +4,7 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" -monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" \ No newline at end of file +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="-" +89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.0.0.2,10.0.0.1" +2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.225.192.17, 10.2.2.121" +monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2" \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 3335a0c69df..345ea54c4b9 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -1,114 +1,134 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200116114Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105740618Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200149914Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105743350Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, - "url": { - "path": "/hello", - "original": "/hello" - }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200158562Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 @@ -118,143 +138,155 @@ "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105743828Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "172.17.0.1" + ] + } }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200165012Z", + "kind": "event", + "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "172.17.0.1", "ip": "172.17.0.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105744250Z", - "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/stringpatch", + "path": "/stringpatch" }, "user": { "name": "-" }, - "url": { - "path": "/stringpatch", - "original": "/stringpatch" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "monitoring-server" + ] + } }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200170453Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, - "event": { - "ingested": "2021-12-14T14:34:05.105744722Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/status", + "path": "/status" }, "user": { "name": "-" }, - "url": { - "path": "/status", - "original": "/status" - }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" - }, - "device": { - "name": "Other" + "version": "7" }, "version": "15.0.a2" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "127.0.0.1" + ] + } }, - "@timestamp": "2019-02-02T04:38:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200175624Z", + "kind": "event", + "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", + "outcome": "failure" + }, "http": { "request": { "referrer": "-" @@ -270,85 +302,311 @@ "address": "127.0.0.1", "ip": "127.0.0.1" }, - "event": { - "ingested": "2021-12-14T14:34:05.105745119Z", - "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, + "tags": [ + "preserve_original_event" + ], "user": { "name": "-" }, "user_agent": { - "name": "Other", "device": { "name": "Other" }, + "name": "Other", "original": "-" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "monitoring-server" + ] + } }, - "@timestamp": "2017-05-29T19:02:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200180593Z", + "kind": "event", + "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 612 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "monitoring-server", "domain": "monitoring-server" }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, + "version": "15.0.a2" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": { + "remote_addresses": [ + "10.0.0.2", + "10.0.0.1", + "89.160.20.112" + ] + } + }, + "client": { + "ip": "10.0.0.2" + }, + "ecs": { + "version": "1.12.0" + }, "event": { - "ingested": "2021-12-14T14:34:05.105745518Z", - "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "category": "web", - "kind": "event", "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200185417Z", + "kind": "event", + "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" }, - "user": { - "name": "-" + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "10.0.0.2" + }, + "source": { + "address": "89.160.20.112", + "as": { + "number": 29518, + "organization": { + "name": "Bredband2 AB" + } + }, + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.112" }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "extension": "mp4", - "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Firefox Alpha", "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", "os": { + "full": "Windows 7", "name": "Windows", - "version": "7", - "full": "Windows 7" + "version": "7" + }, + "version": "15.0.a2" + } + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "apache": { + "access": { + "remote_addresses": [ + "10.225.192.17", + "10.2.2.121", + "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + ] + } + }, + "client": { + "ip": "10.225.192.17" + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200190094Z", + "kind": "event", + "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" }, + "response": { + "body": { + "bytes": 612 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "10.225.192.17" + }, + "source": { + "address": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", + "geo": { + "continent_name": "Europe", + "country_iso_code": "NO", + "country_name": "Norway", + "location": { + "lat": 62.0, + "lon": 10.0 + } + }, + "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "mp4", + "original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4" + }, + "user": { + "name": "-" + }, + "user_agent": { "device": { "name": "Other" }, + "name": "Firefox Alpha", + "original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "os": { + "full": "Windows 7", + "name": "Windows", + "version": "7" + }, "version": "15.0.a2" + } + }, + { + "@timestamp": "2022-05-17T21:41:43.000Z", + "apache": { + "access": { + "remote_addresses": [ + "192.168.0.2", + "monitoring-server" + ] + } + }, + "client": { + "ip": "192.168.0.2" + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.200194779Z", + "kind": "event", + "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET", + "referrer": "-" + }, + "response": { + "body": { + "bytes": 45 + }, + "status_code": 200 + }, + "version": "1.1" + }, + "network": { + "forwarded_ip": "192.168.0.2" + }, + "source": { + "address": "monitoring-server", + "domain": "monitoring-server" }, "tags": [ "preserve_original_event" - ] + ], + "url": { + "original": "/", + "path": "/" + }, + "user": { + "name": "-" + }, + "user_agent": { + "device": { + "name": "Other" + }, + "name": "curl", + "original": "curl/7.79.1", + "version": "7.79.1" + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index b056c3dbb95..e87e7538905 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -1,100 +1,120 @@ { "expected": [ { + "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:28.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405343801Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093531222Z", - "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:29.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405359749Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 209 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093534426Z", - "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, + "tags": [ + "preserve_original_event" + ], "url": { - "path": "/favicon.ico", "extension": "ico", - "original": "/favicon.ico" + "original": "/favicon.ico", + "path": "/favicon.ico" }, - "tags": [ - "preserve_original_event" - ] + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "::1" + ] + } }, - "@timestamp": "2016-12-26T14:16:48.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405365252Z", + "kind": "event", + "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", + "outcome": "failure" + }, "http": { "response": { "status_code": 408 @@ -104,203 +124,207 @@ "address": "::1", "ip": "::1" }, - "event": { - "ingested": "2021-12-14T14:34:06.093534891Z", - "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" - }, - "user": { - "name": "-" - }, "tags": [ "preserve_original_event" - ] + ], + "user": { + "name": "-" + } }, { + "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:35.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405369669Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", + "outcome": "success" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 45 }, "status_code": 200 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093535376Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:41.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405373693Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 206 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093535764Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/notfound", + "path": "/notfound" }, "user": { "name": "-" - }, - "url": { - "path": "/notfound", - "original": "/notfound" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "89.160.20.156" + ] + } }, - "@timestamp": "2016-12-26T16:23:45.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.405377632Z", + "kind": "event", + "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", + "outcome": "failure" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 201 }, "status_code": 404 - } + }, + "version": "1.1" }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.093536142Z", - "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hmm", + "path": "/hmm" }, "user": { "name": "-" - }, - "url": { - "path": "/hmm", - "original": "/hmm" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 8d05c25b3a5..7aa0164b5c2 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -1,117 +1,123 @@ { "expected": [ { + "@timestamp": "2018-08-10T07:45:56.000Z", "apache": { "access": { + "remote_addresses": [ + "172.30.0.119" + ], "ssl": { "cipher": "ECDHE-RSA-AES128-GCM-SHA256", "protocol": "TLSv1.2" } } }, - "@timestamp": "2018-08-10T07:45:56.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.472518977Z", + "kind": "event", + "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" + }, "http": { "request": { "method": "GET" }, - "version": "1.1", "response": { "body": { "bytes": 1375 } - } - }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" + }, + "version": "1.1" }, "source": { "address": "172.30.0.119", "ip": "172.30.0.119" }, - "event": { - "ingested": "2021-12-14T14:34:06.744087534Z", - "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/nagiosxi/ajaxhelper.php", "extension": "php", "original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "path": "/nagiosxi/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026amp;opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" - }, - "tags": [ - "preserve_original_event" - ] + } }, { + "@timestamp": "2019-10-16T09:53:47.000Z", "apache": { "access": { + "remote_addresses": [ + "89.160.20.156" + ], "ssl": { "cipher": "ECDHE-RSA-AES128-GCM-SHA256", "protocol": "TLSv1.2" } } }, - "@timestamp": "2019-10-16T09:53:47.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.472546575Z", + "kind": "event", + "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" + }, "http": { "request": { "method": "GET" }, "version": "1.1" }, - "tls": { - "cipher": "ECDHE-RSA-AES128-GCM-SHA256", - "version": "1.2", - "version_protocol": "tls" - }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:06.744090082Z", - "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z" + "tags": [ + "preserve_original_event" + ], + "tls": { + "cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "version": "1.2", + "version_protocol": "tls" }, "url": { - "path": "/appl/ajaxhelper.php", "extension": "php", "original": "/appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "path": "/appl/ajaxhelper.php", "query": "cmd=getxicoreajax\u0026opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" - }, - "tags": [ - "preserve_original_event" - ] + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 92c297c4b33..122d1046537 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -1,515 +1,551 @@ { "expected": [ { - "source": { - "address": "127.0.0.1", - "ip": "127.0.0.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "127.0.0.1" + ] + } }, - "@timestamp": "2016-12-26T16:18:09.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531606809Z", + "kind": "event", + "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 491 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835525800Z", - "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Other" + }, "name": "Wget", "original": "Wget/1.13.4 (linux-gnu)", "os": { "name": "Linux" }, - "device": { - "name": "Other" - }, "version": "1.13.4" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531626408Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835534600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:00.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531632562Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "http://192.168.33.72/" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835540100Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Chrome", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", "os": { + "full": "Mac OS X 10.12.0", "name": "Mac OS X", - "version": "10.12.0", - "full": "Mac OS X 10.12.0" - }, - "device": { - "name": "Mac" + "version": "10.12.0" }, "version": "54.0.2840.98" } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/", - "original": "/" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531637660Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "success" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 484 }, "status_code": 200 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835543600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "success" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/", + "path": "/" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531642297Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835548Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/favicon.ico", - "extension": "ico", - "original": "/favicon.ico" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:08.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531646911Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 504 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835553700Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "ico", + "original": "/favicon.ico", + "path": "/favicon.ico" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/test", - "original": "/test" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:10.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531651400Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 498 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835559600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/test", + "path": "/test" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/hello", - "original": "/hello" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:13.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531655775Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835563600Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } }, { - "source": { - "address": "192.168.33.1", - "ip": "192.168.33.1" - }, - "url": { - "path": "/crap", - "original": "/crap" - }, - "tags": [ - "preserve_original_event" - ], + "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { - "access": {} + "access": { + "remote_addresses": [ + "192.168.33.1" + ] + } }, - "@timestamp": "2016-12-26T16:22:17.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.531660022Z", + "kind": "event", + "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:31.835568100Z", - "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.1", + "ip": "192.168.33.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "original": "/crap", + "path": "/crap" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 8b6a8cbbef2..d5d7cf93938 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -1,61 +1,66 @@ { "expected": [ { - "destination": { - "domain": "vhost1.domaine.fr" - }, - "source": { - "ip": "192.168.33.2" + "@timestamp": "2016-12-26T16:22:14.000Z", + "apache": { + "access": { + "remote_addresses": [ + "192.168.33.2" + ] + } }, - "url": { - "path": "/hello", - "original": "/hello", + "destination": { "domain": "vhost1.domaine.fr" }, - "tags": [ - "preserve_original_event" - ], - "apache": { - "access": {} - }, - "@timestamp": "2016-12-26T16:22:14.000Z", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2022-07-29T12:32:45.625025669Z", + "kind": "event", + "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", + "outcome": "failure" + }, "http": { "request": { "method": "GET", "referrer": "-" }, - "version": "1.1", "response": { "body": { "bytes": 499 }, "status_code": 404 - } + }, + "version": "1.1" }, - "event": { - "ingested": "2021-12-09T13:30:33.387841500Z", - "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", - "category": "web", - "kind": "event", - "created": "2020-04-28T11:07:58.223Z", - "outcome": "failure" + "source": { + "address": "192.168.33.2", + "ip": "192.168.33.2" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "domain": "vhost1.domaine.fr", + "original": "/hello", + "path": "/hello" }, "user": { "name": "-" }, "user_agent": { + "device": { + "name": "Mac" + }, "name": "Firefox", "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0", "os": { + "full": "Mac OS X 10.12", "name": "Mac OS X", - "version": "10.12", - "full": "Mac OS X 10.12" - }, - "device": { - "name": "Mac" + "version": "10.12" }, "version": "50.0." } diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index dadfb3a4939..e9c332c308b 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -16,20 +16,69 @@ processors: - grok: field: event.original patterns: - - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" - %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( - "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' + "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?( X-Forwarded-For="%{ADDRESS_LIST:apache.access.remote_addresses}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true + pattern_definitions: + ADDRESS_LIST: (%{IP})("?,?\s*(%{IP}))* + - split: + field: apache.access.remote_addresses + separator: '"?,\s*' + ignore_missing: true + - set: + field: network.forwarded_ip + value: "{{{apache.access.remote_addresses.0}}}" + if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 + - script: + if: ctx.apache?.access?.remote_addresses != null && ctx.apache.access.remote_addresses.length > 0 + lang: painless + tag: Get source address + description: Extract from remote_addresses, the first non-private IP to ctx.client.ip + source: >- + boolean isPrivateCIDR(def ip) { + CIDR class_a_network = new CIDR('10.0.0.0/8'); + CIDR class_b_network = new CIDR('172.16.0.0/12'); + CIDR class_c_network = new CIDR('192.168.0.0/16'); + + try { + return class_a_network.contains(ip) || class_b_network.contains(ip) || class_c_network.contains(ip); + } catch (IllegalArgumentException e) { + return false; + } + } + + try { + if (ctx.client == null) { + Map map = new HashMap(); + ctx.put("client", map); + } + + def found = false; + for (def item : ctx.apache.access.remote_addresses) { + if (!isPrivateCIDR(item)) { + ctx.client.ip = item; + found = true; + break; + } + } + if (!found) { + ctx.client.ip = ctx.apache.access.remote_addresses[0]; + } + } + catch (Exception e) { + ctx.client.ip = null; + } + - append: + field: apache.access.remote_addresses + value: ["{{source.address}}"] + if: ctx.source?.address != null - uri_parts: field: _tmp.url_orig ignore_failure: true @@ -50,11 +99,11 @@ processors: - set: field: event.outcome value: success - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code < 400" - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + if: "ctx.http?.response?.status_code != null && ctx.http.response.status_code > 399" - grok: field: source.address ignore_missing: true @@ -102,10 +151,10 @@ processors: - set: field: tls.cipher value: '{{apache.access.ssl.cipher}}' - if: ctx?.apache?.access?.ssl?.cipher != null + if: ctx.apache?.access?.ssl?.cipher != null - script: lang: painless - if: ctx?.apache?.access?.ssl?.protocol != null + if: ctx.apache?.access?.ssl?.protocol != null source: >- def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken("v"); if (parts.length != 2) { @@ -143,7 +192,7 @@ processors: handleMap(ctx); - remove: field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true on_failure: diff --git a/packages/apache/data_stream/access/fields/ecs.yml b/packages/apache/data_stream/access/fields/ecs.yml index 12993b02683..e9c1f6c7d85 100644 --- a/packages/apache/data_stream/access/fields/ecs.yml +++ b/packages/apache/data_stream/access/fields/ecs.yml @@ -1,3 +1,5 @@ +- external: ecs + name: client.ip - external: ecs name: destination.domain - external: ecs @@ -30,6 +32,8 @@ name: log.level - external: ecs name: message +- external: ecs + name: network.forwarded_ip - external: ecs name: process.pid - external: ecs diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index 402fb490e41..f9dc5e7bd4e 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -9,3 +9,8 @@ type: keyword description: | SSL cipher name. + - name: nginx.access + - name: remote_addresses + type: keyword + description: | + An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json index abd455abe8a..adb241ff953 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-basic.log-expected.json @@ -1,170 +1,170 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:45.886133788Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-14T14:34:09.255770595Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-14T14:34:09.255773246Z", - "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", - "type": "info", + "ingested": "2022-07-29T12:32:45.886154200Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 35708, - "thread": { - "id": 4328636416 - } - }, + "@timestamp": "2011-09-09T10:42:29.902+02:00", "apache": { "error": { "module": "core" } }, - "file": { - "path": "/usr/local/apache2/htdocs/favicon.ico" - }, - "@timestamp": "2011-09-09T10:42:29.902+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:45.886161220Z", + "kind": "event", + "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/usr/local/apache2/htdocs/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "process": { + "pid": 35708, + "thread": { + "id": 4328636416 + } + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, "ip": "89.160.20.156" }, - "event": { - "ingested": "2021-12-14T14:34:09.255773777Z", - "original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 89.160.20.156] File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 15934 - }, + "@timestamp": "2019-06-27T06:58:09.169+02:00", "apache": { "error": { "module": "include" } }, - "@timestamp": "2019-06-27T06:58:09.169+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:45.886167076Z", + "kind": "event", + "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "timezone": "GMT+2", + "type": "error" + }, "log": { "level": "warn" }, + "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", + "process": { + "pid": 15934 + }, "source": { - "geo": { - "continent_name": "Europe", - "region_iso_code": "SE-E", - "city_name": "Linköping", - "country_iso_code": "SE", - "country_name": "Sweden", - "region_name": "Östergötland County", - "location": { - "lon": 15.6167, - "lat": 58.4167 - } - }, + "address": "89.160.20.156", "as": { "number": 29518, "organization": { "name": "Bredband2 AB" } }, - "address": "89.160.20.156", - "port": 12345, - "ip": "89.160.20.156" - }, - "event": { - "ingested": "2021-12-14T14:34:09.255774189Z", - "original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 89.160.20.156:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" + "geo": { + "city_name": "Linköping", + "continent_name": "Europe", + "country_iso_code": "SE", + "country_name": "Sweden", + "location": { + "lat": 58.4167, + "lon": 15.6167 + }, + "region_iso_code": "SE-E", + "region_name": "Östergötland County" + }, + "ip": "89.160.20.156", + "port": 12345 }, - "message": "AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json index 15400a6c97a..9c743f595d0 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-darwin.log-expected.json @@ -1,59 +1,59 @@ { "expected": [ { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "mpm_prefork" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.149405700Z", - "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "category": "web", - "type": "info", + "ingested": "2022-07-29T12:32:45.957125294Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] }, { - "process": { - "pid": 11379 - }, + "@timestamp": "2016-12-26T16:15:55.103+02:00", "apache": { "error": { "module": "core" } }, - "@timestamp": "2016-12-26T16:15:55.103+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.149429600Z", - "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "category": "web", - "type": "info", + "ingested": "2022-07-29T12:32:45.957148462Z", + "kind": "event", + "original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", + "process": { + "pid": 11379 + }, "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json index 7b5f6e1b4b1..3c739afc052 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-trace.log-expected.json @@ -1,33 +1,33 @@ { "expected": [ { - "process": { - "pid": 121591, - "thread": { - "id": 140413273032448 - } - }, + "@timestamp": "2021-10-20T19:20:59.121+02:00", "apache": { "error": { "module": "rewrite" } }, - "@timestamp": "2021-10-20T19:20:59.121+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "trace3" - }, "event": { - "ingested": "2021-12-09T13:30:34.228018700Z", - "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "category": "web", - "type": "info", + "ingested": "2022-07-29T12:32:46.015505043Z", + "kind": "event", + "original": "[Wed Oct 20 19:20:59.121211 2021] [rewrite:trace3] [pid 121591:tid 140413273032448] mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "trace3" }, "message": "mod_rewrite.c(470): [client 10.121.192.8:38350] 10.121.192.8 - - [dev.elastic.co/sid#55a374e851c8][rid#7fb438083ac0/initial] applying pattern '^/import/?(.*)$' to uri '/'", + "process": { + "pid": 121591, + "thread": { + "id": 140413273032448 + } + }, "tags": [ "preserve_original_event" ] diff --git a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json index 92c843f5903..2a3ff1117db 100644 --- a/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json +++ b/packages/apache/data_stream/error/_dev/test/pipeline/test-error-ubuntu.log-expected.json @@ -1,23 +1,23 @@ { "expected": [ { + "@timestamp": "2016-12-26T16:17:53.000+02:00", "apache": { "error": {} }, - "@timestamp": "2016-12-26T16:17:53.000+02:00", "ecs": { "version": "1.12.0" }, - "log": { - "level": "notice" - }, "event": { - "ingested": "2021-12-09T13:30:34.283841100Z", - "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "category": "web", - "type": "info", + "ingested": "2022-07-29T12:32:46.067824384Z", + "kind": "event", + "original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "timezone": "GMT+2", - "kind": "event" + "type": "info" + }, + "log": { + "level": "notice" }, "message": "Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "tags": [ @@ -25,192 +25,192 @@ ] }, { + "@timestamp": "2016-12-26T16:22:00.000+02:00", + "apache": { + "error": {} + }, + "ecs": { + "version": "1.12.0" + }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:46.067850374Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, + "http": { + "request": { + "referrer": "http://192.168.33.72/" + } + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "message": "File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "tags": [ "preserve_original_event" - ], + ] + }, + { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:00.000+02:00", "ecs": { "version": "1.12.0" }, - "http": { - "request": { - "referrer": "http://192.168.33.72/" - } - }, "event": { - "ingested": "2021-12-09T13:30:34.283849400Z", - "original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "category": "web", - "type": "error", + "ingested": "2022-07-29T12:32:46.067856458Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "timezone": "GMT+2", - "kind": "event" - } - }, - { - "apache": { - "error": {} + "type": "error" }, "file": { "path": "/var/www/favicon.ico" }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", - "ecs": { - "version": "1.12.0" - }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283853Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:08.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/favicon.ico" - }, - "@timestamp": "2016-12-26T16:22:08.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:46.067861616Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/favicon.ico" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/favicon.ico", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283857200Z", - "original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/favicon.ico", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:10.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/test" - }, - "@timestamp": "2016-12-26T16:22:10.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:46.067866350Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/test" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/test", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283862600Z", - "original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/test", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:13.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/hello" - }, - "@timestamp": "2016-12-26T16:22:13.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:46.067870964Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/hello" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/hello", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283867900Z", - "original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/hello", "tags": [ "preserve_original_event" ] }, { + "@timestamp": "2016-12-26T16:22:17.000+02:00", "apache": { "error": {} }, - "file": { - "path": "/var/www/crap" - }, - "@timestamp": "2016-12-26T16:22:17.000+02:00", "ecs": { "version": "1.12.0" }, + "event": { + "category": "web", + "ingested": "2022-07-29T12:32:46.067875366Z", + "kind": "event", + "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", + "timezone": "GMT+2", + "type": "error" + }, + "file": { + "path": "/var/www/crap" + }, "log": { "level": "error" }, + "message": "File does not exist: /var/www/crap", "source": { "address": "192.168.33.1", "ip": "192.168.33.1" }, - "event": { - "ingested": "2021-12-09T13:30:34.283873300Z", - "original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", - "category": "web", - "type": "error", - "timezone": "GMT+2", - "kind": "event" - }, - "message": "File does not exist: /var/www/crap", "tags": [ "preserve_original_event" ] diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index 2ef8d6fee40..8d1f488eb75 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,8 +19,10 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.ssl.cipher | SSL cipher name. | keyword | +| apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | +| apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | +| client.ip | IP address of the client (IPv4 or IPv6). | ip | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -75,6 +77,7 @@ Access logs collects the Apache access logs. | log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | | process.pid | Process id. | long | | process.thread.id | Thread ID. | long | | source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | @@ -114,6 +117,24 @@ Access logs collects the Apache access logs. | user_agent.version | Version of the user agent. | keyword | +Supported format for the access logs are: + +- [Common Log Format](https://en.wikipedia.org/wiki/Common_Log_Format) + - Defined in apache `LogFormat` by : + >```%h %l %u %t \"%r\" %>s %b``` + - Example: + > `127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326` +- Combined Log Format + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)"``` +- Combined Log Format + X-Forwarded-For header + - Defined in apache `LogFormat` by: + >```%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" X-Forwarded-For=\"%{X-Forwarded-For}i\"``` + - Example: + >```127.0.0.1 user-identifier frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://datawarehouse.us.oracle.com/datamining/contents.htm" "Mozilla/4.7 [en] (WinNT; I)" X-Forwarded-For="10.225.192.17, 10.2.2.121"``` + ### Error Logs Error logs collects the Apache error logs. diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index 4d4696797bf..32e87ccb30a 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: apache title: Apache HTTP Server -version: 1.5.1 +version: 1.6.0 license: basic source: license: Elastic-2.0