diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index db14ba5d23e..38bc4bb2cac 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -326,6 +326,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] - Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] +- Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] *Heartbeat* diff --git a/filebeat/module/nats/log/ingest/pipeline.json b/filebeat/module/nats/log/ingest/pipeline.json deleted file mode 100644 index 0da22c0c8c7..00000000000 --- a/filebeat/module/nats/log/ingest/pipeline.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "description": "Pipeline for parsing nats log logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns":[ - "\\[%{POSINT:process.pid}\\]( %{NATSTIME:nats.log.timestamp})? \\[%{NATSLOGLEVEL:log.level}\\] %{GREEDYDATA:nats.log.info}" - ], - "pattern_definitions": { - "NATSTIME": "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}", - "NATSLOGLEVEL":"(INF|DBG|WRN|ERR|FTL|TRC)" - }, - "ignore_missing": true - } - }, - { - "grok": { - "field": "nats.log.info", - "patterns": [ - "%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:nats.log.client.id} - %{GREEDYDATA:nats.log.msg.info}", - "%{GREEDYDATA:nats.log.msg.data}" - ], - "ignore_missing": true - } - }, - { - "grok": { - "field": "nats.log.msg.info", - "patterns": [ - "%{NATSDIRECTION:network.direction} %{NATSPAYLOAD:nats.log.msg.type}: \\[%{GREEDYDATA:nats.log.msg.payload}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSNOINFO:nats.log.msg.type}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSUNSUB:nats.log.msg.type}\\s+%{POSINT:nats.log.msg.sid}(\\s+%{POSINT:nats.log.msg.max_messages})?\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSPUB:nats.log.msg.type}\\s+%{NOTSPACE:nats.log.msg.subject}(\\s+%{NOTSPACE:nats.log.msg.reply_to})?\\s+%{POSINT:nats.log.msg.bytes}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSSUB:nats.log.msg.type}\\s+%{NOTSPACE:nats.log.msg.subject}(\\s+%{NOTSPACE:nats.log.msg.queue_group})?\\s+%{POSINT:nats.log.msg.sid}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSMSG:nats.log.msg.type}\\s+%{NOTSPACE:nats.log.msg.subject}\\s+%{POSINT:nats.log.msg.sid}(\\s+%{NOTSPACE:nats.log.msg.reply_to})?\\s+%{POSINT:nats.log.msg.bytes}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSCONNECTION:nats.log.msg.type}\\s+%{GREEDYDATA:nats.log.msg.data}\\]", - "%{NATSDIRECTION:network.direction} \\[%{NATSERROR:nats.log.msg.type}\\s+%{GREEDYDATA:nats.log.msg.error\\]", - "%{GREEDYDATA:nats.log.msg.data}" - ], - "pattern_definitions": { - "NATSDIRECTION": "(<<-|->>)", - "NATSMSG": "MSG", - "NATSPUB": "PUB", - "NATSSUB": "SUB", - "NATSUNSUB": "UNSUB", - "NATSPAYLOAD": "MSG_PAYLOAD", - "NATSERROR": "-ERROR", - "NATSPING": "PING", - "NATSPONG": "PONG", - "NATSOK": "OK", - "NATSCONNECT": "CONNECT", - "NATSINFO": "INFO", - "NATSCONNECTION": "(?:%{NATSCONNECT}|%{NATSINFO})", - "NATSNOINFO": "(?:%{NATSPING}|%{NATSPONG}|%{NATSOK})" - }, - "ignore_missing": true - } - }, - { - "remove": { - "field": "nats.log.info" - } - }, - { - "remove": { - "field": "nats.log.msg.info", - "ignore_missing": true - } - }, - { - "remove": { - "field": "nats.log.msg.payload", - "ignore_missing": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "rename": { - "field": "nats.log.msg.data", - "target_field": "message", - "ignore_missing": true - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.log.level == params.inf) {\n ctx.log.level = params.info;\n } else if (ctx.log.level == params.dbg) {\n ctx.log.level = params.debug;\n } else if (ctx.log.level == params.wrn) {\n ctx.log.level = params.warning;\n } else if (ctx.log.level == params.err) {\n ctx.log.level = params.error;\n } else if (ctx.log.level == params.ftl) {\n ctx.log.level = params.fatal;\n } else if (ctx.log.level == params.trc) {\n ctx.log.level = params.trace;\n }", - "params": { - "inf": "INF", - "info": "info", - "dbg": "DBG", - "debug": "debug", - "wrn": "WRN", - "warning": "warning", - "err": "ERR", - "error": "error", - "ftl": "FTL", - "fatal": "fatal", - "trc": "TRC", - "trace": "trace" - } - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.nats.log.msg.type == params.msg) {\n ctx.nats.log.msg.type = params.message;\n } else if (ctx.nats.log.msg.type == params.pub) {\n ctx.nats.log.msg.type = params.publish;\n } else if (ctx.nats.log.msg.type == params.sub) {\n ctx.nats.log.msg.type = params.subscribe;\n } else if (ctx.nats.log.msg.type == params.unsub) {\n ctx.nats.log.msg.type = params.unsubscribe;\n } else if (ctx.nats.log.msg.type == params.msg_payload) {\n ctx.nats.log.msg.type = params.payload;\n } else if (ctx.nats.log.msg.type == params.err) {\n ctx.nats.log.msg.type = params.error;\n } else if (ctx.nats.log.msg.type == params.pi) {\n ctx.nats.log.msg.type = params.ping;\n } else if (ctx.nats.log.msg.type == params.po) {\n ctx.nats.log.msg.type = params.pong;\n } else if (ctx.nats.log.msg.type == params.ok) {\n ctx.nats.log.msg.type = params.acknowledge;\n } else if (ctx.nats.log.msg.type == params.connect) {\n ctx.nats.log.msg.type = params.connection;\n } else if (ctx.nats.log.msg.type == params.info) {\n ctx.nats.log.msg.type = params.information;\n }", - "params": { - "msg": "MSG", - "message": "message", - "pub": "PUB", - "publish": "publish", - "sub": "SUB", - "subscribe": "subscribe", - "unsub": "UNSUB", - "unsubscribe": "unsubscribe", - "msg_payload": "MSG_PAYLOAD", - "payload": "payload", - "err": "-ERROR", - "error": "error", - "pi": "PING", - "ping": "ping", - "po": "PONG", - "pong": "pong", - "ok": "OK", - "acknowledge": "acknowledge", - "connect": "CONNECT", - "connection": "connection", - "info": "INFO", - "information": "information" - }, - "if": "ctx.nats.log.msg?.type != null" - } - }, - { - "script": { - "lang": "painless", - "source": "if (ctx.network.direction == params.in) {\n ctx.network.direction = params.inbound;\n } else if (ctx.network.direction == params.out) {\n ctx.network.direction = params.outbound;\n }", - "params": { - "in": "<<-", - "inbound": "inbound", - "out": "->>", - "outbound": "outbound" - }, - "if": "ctx.network?.direction != null" - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "nats.log.timestamp", - "target_field": "@timestamp", - "formats": ["yyyy/MM/dd HH:mm:ss.SSSSSS"] - } - }, - { - "remove": { - "field": "nats.log.timestamp" - } - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml new file mode 100644 index 00000000000..53c4f774b5e --- /dev/null +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -0,0 +1,181 @@ +description: Pipeline for parsing nats log logs +processors: +- grok: + field: message + patterns: + - \[%{POSINT:process.pid}\]( %{NATSTIME:nats.log.timestamp})? \[%{NATSLOGLEVEL:log.level}\] + %{GREEDYDATA:nats.log.info} + pattern_definitions: + NATSTIME: '%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}' + NATSLOGLEVEL: (INF|DBG|WRN|ERR|FTL|TRC) + ignore_missing: true +- grok: + field: nats.log.info + patterns: + - '%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:nats.log.client.id} + - %{GREEDYDATA:nats.log.msg.info}' + - '%{GREEDYDATA:nats.log.msg.data}' + ignore_missing: true +- grok: + field: nats.log.msg.info + patterns: + - '%{NATSDIRECTION:network.direction} %{NATSPAYLOAD:nats.log.msg.type}: \[%{GREEDYDATA:nats.log.msg.payload}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSNOINFO:nats.log.msg.type}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSUNSUB:nats.log.msg.type}\s+%{POSINT:nats.log.msg.sid}(\s+%{POSINT:nats.log.msg.max_messages})?\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSPUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSSUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.queue_group})?\s+%{POSINT:nats.log.msg.sid}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSMSG:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}\s+%{POSINT:nats.log.msg.sid}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSCONNECTION:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.data}\]' + - '%{NATSDIRECTION:network.direction} \[%{NATSERROR:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.error\]' + - '%{GREEDYDATA:nats.log.msg.data}' + pattern_definitions: + NATSDIRECTION: (<<-|->>) + NATSMSG: MSG + NATSPUB: PUB + NATSSUB: SUB + NATSUNSUB: UNSUB + NATSPAYLOAD: MSG_PAYLOAD + NATSERROR: -ERROR + NATSPING: PING + NATSPONG: PONG + NATSOK: OK + NATSCONNECT: CONNECT + NATSINFO: INFO + NATSCONNECTION: (?:%{NATSCONNECT}|%{NATSINFO}) + NATSNOINFO: (?:%{NATSPING}|%{NATSPONG}|%{NATSOK}) + ignore_missing: true +- remove: + field: nats.log.info +- remove: + field: nats.log.msg.info + ignore_missing: true +- remove: + field: nats.log.msg.payload + ignore_missing: true +- remove: + field: message +- rename: + field: nats.log.msg.data + target_field: message + ignore_missing: true +- script: + lang: painless + source: |- + if (ctx.log.level == params.inf) { + ctx.log.level = params.info; + } else if (ctx.log.level == params.dbg) { + ctx.log.level = params.debug; + } else if (ctx.log.level == params.wrn) { + ctx.log.level = params.warning; + } else if (ctx.log.level == params.err) { + ctx.log.level = params.error; + } else if (ctx.log.level == params.ftl) { + ctx.log.level = params.fatal; + } else if (ctx.log.level == params.trc) { + ctx.log.level = params.trace; + } + params: + inf: INF + info: info + dbg: DBG + debug: debug + wrn: WRN + warning: warning + err: ERR + error: error + ftl: FTL + fatal: fatal + trc: TRC + trace: trace +- script: + lang: painless + source: |- + if (ctx.nats.log.msg.type == params.msg) { + ctx.nats.log.msg.type = params.message; + } else if (ctx.nats.log.msg.type == params.pub) { + ctx.nats.log.msg.type = params.publish; + } else if (ctx.nats.log.msg.type == params.sub) { + ctx.nats.log.msg.type = params.subscribe; + } else if (ctx.nats.log.msg.type == params.unsub) { + ctx.nats.log.msg.type = params.unsubscribe; + } else if (ctx.nats.log.msg.type == params.msg_payload) { + ctx.nats.log.msg.type = params.payload; + } else if (ctx.nats.log.msg.type == params.err) { + ctx.nats.log.msg.type = params.error; + } else if (ctx.nats.log.msg.type == params.pi) { + ctx.nats.log.msg.type = params.ping; + } else if (ctx.nats.log.msg.type == params.po) { + ctx.nats.log.msg.type = params.pong; + } else if (ctx.nats.log.msg.type == params.ok) { + ctx.nats.log.msg.type = params.acknowledge; + } else if (ctx.nats.log.msg.type == params.connect) { + ctx.nats.log.msg.type = params.connection; + } else if (ctx.nats.log.msg.type == params.info) { + ctx.nats.log.msg.type = params.information; + } + params: + msg: MSG + message: message + pub: PUB + publish: publish + sub: SUB + subscribe: subscribe + unsub: UNSUB + unsubscribe: unsubscribe + msg_payload: MSG_PAYLOAD + payload: payload + err: -ERROR + error: error + pi: PING + ping: ping + po: PONG + pong: pong + ok: OK + acknowledge: acknowledge + connect: CONNECT + connection: connection + info: INFO + information: information + if: ctx.nats.log.msg?.type != null +- script: + lang: painless + source: |- + if (ctx.network.direction == params.in) { + ctx.network.direction = params.inbound; + } else if (ctx.network.direction == params.out) { + ctx.network.direction = params.outbound; + } + params: + in: <<- + inbound: inbound + out: ->> + outbound: outbound + if: ctx.network?.direction != null +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: nats.log.timestamp + target_field: '@timestamp' + formats: + - yyyy/MM/dd HH:mm:ss.SSSSSS +- remove: + field: nats.log.timestamp +- set: + field: event.kind + value: event +- append: + field: event.type + value: info +- append: + field: event.type + value: error + if: "ctx?.log?.level != null && (ctx.log.level == 'error' || ctx.log.level == 'fatal')" +- append: + field: related.ip + value: "{{client.ip}}" + if: "ctx?.client?.ip != null" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/nats/log/manifest.yml b/filebeat/module/nats/log/manifest.yml index 8c51b92a137..86c0d041cb2 100644 --- a/filebeat/module/nats/log/manifest.yml +++ b/filebeat/module/nats/log/manifest.yml @@ -11,5 +11,5 @@ var: # See more on https://nats.io/documentation/server/gnatsd-logging/ - /var/log/nats/nats.log* -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/log.yml diff --git a/filebeat/module/nats/log/test/test.log-expected.json b/filebeat/module/nats/log/test/test.log-expected.json index 34dd0c2f17f..efa4fba7509 100644 --- a/filebeat/module/nats/log/test/test.log-expected.json +++ b/filebeat/module/nats/log/test/test.log-expected.json @@ -2,7 +2,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -14,7 +18,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -26,7 +34,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -38,7 +50,11 @@ { "@timestamp": "2019-02-06T07:19:40.624Z", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "info", @@ -52,7 +68,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "debug", @@ -60,6 +80,9 @@ "message": "Client connection created", "nats.log.client.id": "1", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -67,7 +90,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -77,6 +104,9 @@ "nats.log.msg.type": "connection", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -84,7 +114,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -95,6 +129,9 @@ "nats.log.msg.type": "subscribe", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -102,7 +139,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -111,6 +152,9 @@ "nats.log.msg.type": "ping", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -118,7 +162,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -127,6 +175,9 @@ "nats.log.msg.type": "pong", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" }, { @@ -134,7 +185,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -146,6 +201,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -153,7 +211,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -162,6 +224,9 @@ "nats.log.msg.type": "payload", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -169,7 +234,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -182,6 +251,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -189,7 +261,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -200,6 +276,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -207,7 +286,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -216,6 +299,9 @@ "nats.log.msg.type": "payload", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -223,7 +309,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -235,6 +325,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -242,7 +335,11 @@ "client.ip": "50.39.246.116", "client.port": "62388", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -253,6 +350,9 @@ "nats.log.msg.type": "publish", "network.direction": "outbound", "process.pid": "1", + "related.ip": [ + "50.39.246.116" + ], "service.type": "nats" }, { @@ -260,7 +360,11 @@ "client.ip": "192.168.176.11", "client.port": "36262", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -272,6 +376,9 @@ "nats.log.msg.type": "message", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "192.168.176.11" + ], "service.type": "nats" }, { @@ -279,7 +386,11 @@ "client.ip": "172.18.0.1", "client.port": "38630", "event.dataset": "nats.log", + "event.kind": "event", "event.module": "nats", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "trace", @@ -288,6 +399,9 @@ "nats.log.msg.type": "acknowledge", "network.direction": "inbound", "process.pid": "1", + "related.ip": [ + "172.18.0.1" + ], "service.type": "nats" } ] \ No newline at end of file