diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e3cdf31ac23b..2a650c944256 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -39,6 +39,7 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef *Filebeat* - Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869] +- Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958] *Heartbeat* diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index b0ae63df4a85..f409ec5ba7df 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -7,7 +7,7 @@ "patterns": [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:haproxy.process_name}(?:\\[%{POSINT:haproxy.pid}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:haproxy.client.port} %{WORD} %{IPORHOST:haproxy.destination.ip}:%{POSINT:haproxy.destination.port} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", @@ -68,4 +68,4 @@ } } ] -} \ No newline at end of file +} diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log b/filebeat/module/haproxy/log/test/httplog-no-headers.log new file mode 100644 index 000000000000..e6d4f96f4b74 --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log @@ -0,0 +1,4 @@ +Dec 10 12:01:46 voyager haproxy[19312]: 127.0.0.1:35982 [10/Dec/2018:12:01:46.395] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 "GET / HTTP/1.1" +Dec 10 15:46:49 voyager haproxy[29785]: 127.0.0.1:43738 [10/Dec/2018:15:46:49.497] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} "GET /foo HTTP/1.1" +Dec 10 15:48:56 voyager haproxy[7873]: 127.0.0.1:44542 [10/Dec/2018:15:48:56.017] http-webservices http-webservices/ 0/-1/-1/-1/0 503 213 - - SC-- 1/1/0/0/0 0/0 {localhost:8888||} {|} "GET /foo HTTP/1.1" + diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json new file mode 100644 index 000000000000..4415959511f8 --- /dev/null +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -0,0 +1,105 @@ +[ + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 35982, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.pid": 19312, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 0, + "prospector.type": "log" + }, + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 43738, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.status_code": 503, + "haproxy.pid": 29785, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 186, + "prospector.type": "log" + }, + { + "event.dataset": "haproxy.log", + "fileset.module": "haproxy", + "fileset.name": "log", + "haproxy.backend_name": "http-webservices", + "haproxy.backend_queue": 0, + "haproxy.bytes_read": 213, + "haproxy.client.ip": "127.0.0.1", + "haproxy.client.port": 44542, + "haproxy.connection_wait_time_ms": -1, + "haproxy.connections.active": 1, + "haproxy.connections.backend": 0, + "haproxy.connections.frontend": 1, + "haproxy.connections.retries": 0, + "haproxy.connections.server": 0, + "haproxy.frontend_name": "http-webservices", + "haproxy.http.request.captured_cookie": "-", + "haproxy.http.request.captured_headers": [ + "localhost:8888" + ], + "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", + "haproxy.http.request.time_active_ms": 0, + "haproxy.http.request.time_wait_ms": 0, + "haproxy.http.request.time_wait_without_data_ms": -1, + "haproxy.http.response.captured_cookie": "-", + "haproxy.http.response.captured_headers": [], + "haproxy.http.response.status_code": 503, + "haproxy.pid": 7873, + "haproxy.process_name": "haproxy", + "haproxy.server_name": "", + "haproxy.server_queue": 0, + "haproxy.termination_state": "SC--", + "haproxy.total_waiting_time_ms": -1, + "input.type": "log", + "offset": 394, + "prospector.type": "log" + } +] \ No newline at end of file