From d97d603ae7653857212d28dacb2dfd18764faf04 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Thu, 10 Jan 2019 21:51:11 +0100 Subject: [PATCH] Support haproxy log lines without captured headers (#9958) (#9968) Haproxy can capture headers from http requests and responses and log them. This is not done by default but current filebeat module expects it. Make captured headers optional, and collect them only if both request and response headers are configured. If only one is configured, the log is parsed but headers not collected as we cannot know if they are request or response headers. (cherry picked from commit b39d780cb54c25299c57db9cfd68e809269e71b3) --- CHANGELOG.next.asciidoc | 1 + .../module/haproxy/log/ingest/pipeline.json | 4 +- .../haproxy/log/test/httplog-no-headers.log | 4 + .../test/httplog-no-headers.log-expected.json | 105 ++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 filebeat/module/haproxy/log/test/httplog-no-headers.log create mode 100644 filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e8d9f00b8cf..b7b7ce94599 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -70,6 +70,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Fixed a memory leak when harvesters are closed. {pull}7820[7820] - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761] - Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869] +- Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958] *Heartbeat* diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index b0ae63df4a8..f409ec5ba7d 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -7,7 +7,7 @@ "patterns": [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:haproxy.process_name}(?:\\[%{POSINT:haproxy.pid}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:haproxy.client.port} %{WORD} %{IPORHOST:haproxy.destination.ip}:%{POSINT:haproxy.destination.port} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", @@ -68,4 +68,4 @@ } } ] -} \ No newline at end of file +} diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log b/filebeat/module/haproxy/log/test/httplog-no-headers.log new file mode 100644 index 00000000000..e6d4f96f4b7 --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log @@ -0,0 +1,4 @@ +Dec 10 12:01:46 voyager haproxy[19312]: 127.0.0.1:35982 [10/Dec/2018:12:01:46.395] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 "GET / HTTP/1.1" +Dec 10 15:46:49 voyager haproxy[29785]: 127.0.0.1:43738 [10/Dec/2018:15:46:49.497] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} "GET /foo HTTP/1.1" +Dec 10 15:48:56 voyager haproxy[7873]: 127.0.0.1:44542 [10/Dec/2018:15:48:56.017] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} {|} "GET /foo HTTP/1.1" + diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json new file mode 100644 index 00000000000..084a2c631ab --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -0,0 +1,105 @@ +[ + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 35982, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.pid": 19312, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 0, + "prospector.type": "log" + }, + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 43738, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.pid": 29785, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 186, + "prospector.type": "log" + }, + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 44542, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.captured_headers": [ + "localhost:8888" + ], + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.captured_headers": [], + "haproxy.http.response.status_code": 503, + "haproxy.pid": 7873, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 394, + "prospector.type": "log" + } +] \ No newline at end of file