From bf19e279825ae11428b63b5349313b59cfe53279 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 20 Feb 2019 23:52:44 -0500 Subject: [PATCH 1/3] Enhance Suricata pipeline to handle destination.domain being set This replaces the usage of a `rename` processor with an `append` + `remove` processor. Then a script processor is used to deduplicate the domains. Fixes #10510 --- CHANGELOG.next.asciidoc | 1 + .../module/suricata/eve/ingest/pipeline.json | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0880a81784f1..6895092e8415 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -252,6 +252,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877] - Set `event.category: network_traffic` for Suricata. {pull}10882[10882] - Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911] +- Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510] *Heartbeat* diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index e1fcf7c0c6e5..5d0d7cc921f6 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -50,10 +50,22 @@ } }, { - "rename": { + "append": { + "if": "ctx.suricata?.eve?.http?.hostname != null", + "value": "{{suricata.eve.http.hostname}}", + "field": "destination.domain" + } + }, + { + "remove": { "field": "suricata.eve.http.hostname", - "target_field": "destination.domain", - "ignore_missing": true + "ignore_failure": true + } + }, + { + "script": { + "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { if (domain.length == 1) { ctx.destination.domain = domain[0]; } else { ctx.destination.domain = ctx.destination.domain.stream().distinct().collect(Collectors.toList()); } }", + "ignore_failure": true } }, { From ee3aed3e44d0826b1dfbb60043ae6dbd7ad7cb79 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 22 Feb 2019 16:40:13 -0500 Subject: [PATCH 2/3] De-duplicate first --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 5d0d7cc921f6..946cda87a353 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -64,7 +64,7 @@ }, { "script": { - "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { if (domain.length == 1) { ctx.destination.domain = domain[0]; } else { ctx.destination.domain = ctx.destination.domain.stream().distinct().collect(Collectors.toList()); } }", + "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { ctx.destination.domain = ctx.destination.domain.stream().distinct().collect(Collectors.toList()); if (domain.length == 1) { ctx.destination.domain = domain[0]; } }", "ignore_failure": true } }, From a95e713c36b6559716412634c02cdde7f4c1722f Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 25 Feb 2019 15:53:02 -0500 Subject: [PATCH 3/3] Fix deduplication script --- x-pack/filebeat/module/suricata/eve/ingest/pipeline.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json index 946cda87a353..3276c1968db7 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.json @@ -64,7 +64,8 @@ }, { "script": { - "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { ctx.destination.domain = ctx.destination.domain.stream().distinct().collect(Collectors.toList()); if (domain.length == 1) { ctx.destination.domain = domain[0]; } }", + "type": "painless", + "source": "def domain = ctx.destination?.domain; if (domain instanceof Collection) { domain = domain.stream().distinct().collect(Collectors.toList()); if (domain.length == 1) { domain = domain[0]; }ctx.destination.domain = domain; }", "ignore_failure": true } },