From 252ff585aa6e41bd08e614db179facd9d102b8ee Mon Sep 17 00:00:00 2001 From: leweafan Date: Sat, 4 Mar 2023 17:31:35 +0300 Subject: [PATCH 1/2] Add unix socket log parsing for nginx ingress_controller --- CHANGELOG.next.asciidoc | 1 + .../ingress_controller/ingest/pipeline.yml | 2 +- .../nginx/ingress_controller/test/test.log | 1 + .../test/test.log-expected.json | 61 +++++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c275d65e6e1..80619a39404 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -223,6 +223,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Fixed GCS log format issues. {pull}34659[34659] - Add nginx.ingress_controller.upstream.ip to related.ip {issue}34645[34645] {pull}34672[34672] - Include NAT and firewall IPs in `related.ip` in Fortinet Firewall module. {issue}34640[34640] {pull}34673[34673] +- Add unix socket log parsing for nginx ingress_controller *Auditbeat* diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index ca000547e90..10d722a98a3 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -23,7 +23,7 @@ processors: NGINX_HOST: (?:%{IP:destination.ip}|%{NGINX_NOTSEPARATOR:destination.domain})(:%{NUMBER:destination.port})? NGINX_NOTSEPARATOR: "[^\t ,:]+" NGINX_ADDRESS_LIST: (?:%{IP}|%{WORD})("?,?\s*(?:%{IP}|%{WORD}))* - UPSTREAM_ADDRESS_LIST: (?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))* + UPSTREAM_ADDRESS_LIST: (unix:%{NOTSPACE}|(?:%{IP}(:%{NUMBER})?)("?,?\s*(?:%{IP}(:%{NUMBER})?))*) UPSTREAM_RESPONSE_LENGTH_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* UPSTREAM_RESPONSE_TIME_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* UPSTREAM_RESPONSE_STATUS_CODE_LIST: (?:%{NUMBER})("?,?\s*(?:%{NUMBER}))* diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index 1c80c68213b..3e9090fe2ec 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -25,3 +25,4 @@ 2a02:cf40:: - remote_monitoring_user [24/Aug/2022:21:04:17 +0000] "POST /_bulk HTTP/1.1" 200 470 "-" "Elastic-metricbeat/7.16.3 (linux; amd64; e7cede6a62ed4452bd9044fc6f4947df; 2022-01-07 00:50:33 +0000 UTC)" 2057 0.033 [esmon-esmon-es-http-9200] [] [2a02:cf40::]:9200 470 0.036 200 3db73c6c673c4256ade033a6ce08c2ab 2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::4e36]:5000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab 2a02:cf40::4e36 - - [24/Aug/2022:18:05:41 +0000] "GET /favicon.ico HTTP/2.0" 502 552 "https://localhost:8080/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" 27 0.000 [localhost-8080] [] [2a02:cf40::7]:5000, [2a02:cf40::::::::4e36]:500000000 0, 0 0.000, 0.000 502, 502 3db73c6c673c4256ade033a6ce08c2ab +192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] "GET /products/42 HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15" 369 0.002 [default-web-8080] [] unix:/var/run/php-fpm.sock 59 0.002 200 0f76ea730f282d5759018eb756b23b14 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index 77a2918fd02..75a611dbb90 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -1702,5 +1702,66 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.15.7", "user_agent.version": "104.0.0.0" + }, + { + "@timestamp": "2020-02-07T11:56:54.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.002 [default-web-8080] [] unix:/var/run/php-fpm.sock 59 0.002 200 0f76ea730f282d5759018eb756b23b14", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "0f76ea730f282d5759018eb756b23b14", + "http.request.method": "GET", + "http.response.body.bytes": 59, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 7430, + "nginx.ingress_controller.http.request.id": "0f76ea730f282d5759018eb756b23b14", + "nginx.ingress_controller.http.request.length": 369, + "nginx.ingress_controller.http.request.time": 0.002, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.1" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.name": "default-web-8080", + "nginx.ingress_controller.upstream.response.length": 59, + "nginx.ingress_controller.upstream.response.length_list": [ + "59" + ], + "nginx.ingress_controller.upstream.response.status_code": 200, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200" + ], + "nginx.ingress_controller.upstream.response.time": 0.002, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.002" + ], + "nginx.ingress_controller.upstream_address_list": [ + "unix:/var/run/php-fpm.sock" + ], + "related.ip": [ + "192.168.64.1" + ], + "service.type": "nginx", + "source.address": "192.168.64.1", + "source.ip": "192.168.64.1", + "url.original": "/products/42", + "url.path": "/products/42", + "user_agent.device.name": "Mac", + "user_agent.name": "Safari", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", + "user_agent.os.full": "Mac OS X 10.14.6", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14.6", + "user_agent.version": "13.0.5" } ] From 272614bb5f02747e7e751e0fc8fd5e020dfcb573 Mon Sep 17 00:00:00 2001 From: Alexander A Date: Mon, 6 Mar 2023 21:29:23 +0300 Subject: [PATCH 2/2] Update CHANGELOG.next.asciidoc - added pull request id Co-authored-by: Tetiana Kravchenko --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ba6bd471da3..0d4f2662478 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -224,7 +224,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415 - Fixed GCS log format issues. {pull}34659[34659] - Add nginx.ingress_controller.upstream.ip to related.ip {issue}34645[34645] {pull}34672[34672] - Include NAT and firewall IPs in `related.ip` in Fortinet Firewall module. {issue}34640[34640] {pull}34673[34673] -- Add unix socket log parsing for nginx ingress_controller +- Add unix socket log parsing for nginx ingress_controller {pull}34732[34732] *Auditbeat*