diff --git a/filebeat/module/apache2/access/ingest/default.json b/filebeat/module/apache2/access/ingest/default.json index fa112c470f3..d916b0ae043 100644 --- a/filebeat/module/apache2/access/ingest/default.json +++ b/filebeat/module/apache2/access/ingest/default.json @@ -4,8 +4,8 @@ "grok": { "field": "message", "patterns":[ - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:int} (?:%{NUMBER:apache2.access.body_sent.bytes:int}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache2.access.agent}\")?", - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache2.access.time}\\] \"-\" %{NUMBER:http.response.status_code:int} -" + "%{IPORHOST:apache2.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:apache2.access.body_sent.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:apache2.access.agent}\")?", + "%{IPORHOST:apache2.access.remote_ip} - %{DATA:user.name} \\[%{HTTPDATE:apache2.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -" ], "ignore_missing": true } diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index 23ac50c334c..e55c38c7a86 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -5,13 +5,13 @@ "grok": { "field": "message", "patterns": [ - "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:int} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:int} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", + "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:haproxy.http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", - "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.tcp.processing_time_ms:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int}" + "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" ], "ignore_missing": false, "pattern_definitions": { diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json index 26d4973c868..3a9a8fdc8b5 100644 --- a/filebeat/module/iis/access/ingest/default.json +++ b/filebeat/module/iis/access/ingest/default.json @@ -4,9 +4,9 @@ "grok": { "field": "message", "patterns":[ - "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.request_time_ms:int}", - "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.body_sent.bytes:int} %{NUMBER:iis.access.body_received.bytes:int} %{NUMBER:iis.access.request_time_ms:int}", - "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} HTTP/%{NUMBER:http.version} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.body_sent.bytes:int} %{NUMBER:iis.access.body_received.bytes:int} %{NUMBER:iis.access.request_time_ms:int}" + "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:iis.access.request_time_ms:long}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:iis.access.body_sent.bytes:long} %{NUMBER:iis.access.body_received.bytes:long} %{NUMBER:iis.access.request_time_ms:long}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.ip} HTTP/%{NUMBER:http.version} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:iis.access.body_sent.bytes:long} %{NUMBER:iis.access.body_received.bytes:long} %{NUMBER:iis.access.request_time_ms:long}" ], "ignore_missing": true } diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index be4398272a9..d4c789343ef 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:int} %{NUMBER:nginx.access.body_sent.bytes:int} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\"" + "\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:nginx.access.body_sent.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\"" ], "pattern_definitions": { "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json index 297167d3f28..6fdd2189bc6 100644 --- a/filebeat/module/system/auth/ingest/pipeline.json +++ b/filebeat/module/system/auth/ingest/pipeline.json @@ -9,13 +9,13 @@ "GREEDYMULTILINE" : "(.|\n)*" }, "patterns": [ - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:int}\\])?: %{DATA:event.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:int} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:int}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:int}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:int}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:int}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:int}", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:int}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:int}, GID=%{NUMBER:group.id:int}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$", - "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYMULTILINE:system.auth.message}" + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$", + "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.auth.message}" ] } }, diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json index ff228a66c30..87a29129cc8 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.json +++ b/filebeat/module/system/syslog/ingest/pipeline.json @@ -5,7 +5,7 @@ "grok": { "field": "message", "patterns": [ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYMULTILINE:system.syslog.message}", + "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.syslog.message}", "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" ], "pattern_definitions" : {