From b3ff82a47795f79536a06e339fd426aa648d7914 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 22 Feb 2021 21:09:13 +0100 Subject: [PATCH 1/2] Fix errors from aws/vpcflow ingest pipeline The pipeline failed with an obscure error.message: `Cannot invoke "Object.getClass()" because "receiver" is null` when the ingested message didn't contain `aws.vpcflow.*` fiels. Filebeat generates documents that lack those fields when parsing lines from a .log file (not .json) that doesn't conform to the expected formats. This happened for empty lines in particular. --- .../module/aws/vpcflow/ingest/pipeline.yml | 8 +++----- .../filebeat/module/aws/vpcflow/test/bad.log | 1 + .../aws/vpcflow/test/bad.log-expected.json | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/bad.log create mode 100644 x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index 0a87d6baaded..2ce2d4a1ad71 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -119,7 +119,7 @@ processors: ignore_empty_value: true - set: - if: "ctx.aws.vpcflow.instance_id != '-'" + if: "ctx.aws?.vpcflow?.instance_id != null && ctx.aws.vpcflow.instance_id != '-'" field: cloud.instance.id value: "{{aws.vpcflow.instance_id}}" ignore_empty_value: true @@ -131,11 +131,9 @@ processors: - script: lang: painless ignore_failure: true + if: "ctx.aws?.vpcflow?.tcp_flags != null" source: | - if (ctx?.aws?.vpcflow?.tcp_flags == null) - return; - - if (ctx?.aws?.vpcflow?.tcp_flags_array == null) { + if (ctx.aws.vpcflow.tcp_flags_array == null) { ArrayList al = new ArrayList(); ctx.aws.vpcflow.put("tcp_flags_array", al); } diff --git a/x-pack/filebeat/module/aws/vpcflow/test/bad.log b/x-pack/filebeat/module/aws/vpcflow/test/bad.log new file mode 100644 index 000000000000..6ac4ad6fc476 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/bad.log @@ -0,0 +1 @@ +Phony unsupported log format. diff --git a/x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json b/x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json new file mode 100644 index 000000000000..534c05beba51 --- /dev/null +++ b/x-pack/filebeat/module/aws/vpcflow/test/bad.log-expected.json @@ -0,0 +1,18 @@ +[ + { + "cloud.provider": "aws", + "event.category": "network_traffic", + "event.dataset": "aws.vpcflow", + "event.kind": "event", + "event.module": "aws", + "event.original": "Phony unsupported log format.", + "event.type": "flow", + "fileset.name": "vpcflow", + "input.type": "log", + "log.offset": 0, + "service.type": "aws", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file From bc00e65d95bedb5f10090267cc298178a17eed9f Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 22 Feb 2021 21:21:38 +0100 Subject: [PATCH 2/2] changelog --- CHANGELOG.next.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 10b788744bc2..21c1d90a394b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -387,7 +387,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix Logstash module handling of logstash.log.log_event.action field. {issue}20709[20709] - aws/s3access dataset was populating event.duration using the wrong unit. {pull}23920[23920] - Zoom module pipeline failed to ingest some chat_channel events. {pull}23904[23904] -- Fix Netlow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110] +- Fix Netflow module issue with missing `internal_networks` config parameter. {issue}24094[24094] {pull}24110[24110] +- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167] *Heartbeat*