From 7797e2cdd72017cb8040daa4f2625507e44ef10f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 1 Apr 2020 16:52:47 -0700 Subject: [PATCH] Add pattern to handle logs from newer ES versions (#17406) * Add pattern to handle logs from newer ES versions * Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + .../audit/ingest/pipeline-json.yml | 1 + .../audit/test/test-audit-761.log | 1 + .../test/test-audit-761.log-expected.json | 46 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-761.log create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d4c54cbc4d8..925f3b02cd5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220] - Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243] - Fixed MySQL slowlog module causing "regular expression has redundant nested repeat operator" warning in Elasticsearch. {issue}17086[17086] {pull}17156[17156] +- Fix `elasticsearch.audit` data ingest pipeline to be more forgiving with date formats found in Elasticsearch audit logs. {pull}17406[17406] *Heartbeat* diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml index 434db5cab21..45a096931f8 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -14,6 +14,7 @@ processors: target_field: elasticsearch.audit.@timestamp formats: - yyyy-MM-dd'T'HH:mm:ss,SSS + - yyyy-MM-dd'T'HH:mm:ss,SSSZ timezone: '{{ event.timezone }}' - remove: if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log b/filebeat/module/elasticsearch/audit/test/test-audit-761.log new file mode 100644 index 00000000000..c3e9a5d3452 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-761.log @@ -0,0 +1 @@ +{"@timestamp":"2020-04-01T11:21:06,725+0200", "node.id":"vvj136QVQ2Ci2aXmrhyi3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"logstash_manager", "user.realm":"native1", "user.roles":["logstash_admin","cluster_monitor"], "origin.type":"rest", "origin.address":"10.54.25.111:52148", "request.id":"rLBMfPM2Q9q-DQEB_g30ww", "action":"indices:data/read/mget[shard]", "request.name":"MultiGetShardRequest", "indices":[".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash"]} diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json new file mode 100644 index 00000000000..c2bb0468065 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json @@ -0,0 +1,46 @@ +[ + { + "@timestamp": "2020-04-01T13:21:06.725Z", + "elasticsearch.audit.action": "indices:data/read/mget[shard]", + "elasticsearch.audit.indices": [ + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "rLBMfPM2Q9q-DQEB_g30ww", + "elasticsearch.audit.request.name": "MultiGetShardRequest", + "elasticsearch.audit.user.realm": "native1", + "elasticsearch.audit.user.roles": [ + "logstash_admin", + "cluster_monitor" + ], + "elasticsearch.node.id": "vvj136QVQ2Ci2aXmrhyi3Q", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "host.id": "vvj136QVQ2Ci2aXmrhyi3Q", + "input.type": "log", + "log.offset": 0, + "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}", + "related.user": [ + "logstash_manager" + ], + "service.type": "elasticsearch", + "source.address": "10.54.25.111:52148", + "source.ip": "10.54.25.111", + "source.port": 52148, + "user.name": "logstash_manager" + } +] \ No newline at end of file