From 55bf5f0d94457a3e5b37d14a6322784382e26f85 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 22 Oct 2020 11:34:32 -0600 Subject: [PATCH] Add max_number_of_messages into aws filebeat fileset vars (#22057) (cherry picked from commit 5d077092d3e0aacfecae81ea307a3c6fda748705) --- x-pack/filebeat/filebeat.reference.yml | 36 +++++++++++++++++++ x-pack/filebeat/module/aws/_meta/config.yml | 36 +++++++++++++++++++ .../module/aws/cloudtrail/config/s3.yml | 4 +++ .../module/aws/cloudtrail/manifest.yml | 2 ++ .../module/aws/cloudwatch/config/s3.yml | 4 +++ .../module/aws/cloudwatch/manifest.yml | 2 ++ x-pack/filebeat/module/aws/ec2/config/s3.yml | 4 +++ x-pack/filebeat/module/aws/ec2/manifest.yml | 2 ++ x-pack/filebeat/module/aws/elb/config/s3.yml | 4 +++ x-pack/filebeat/module/aws/elb/manifest.yml | 2 ++ .../module/aws/s3access/config/s3.yml | 4 +++ .../filebeat/module/aws/s3access/manifest.yml | 2 ++ .../module/aws/vpcflow/config/input.yml | 4 +++ .../filebeat/module/aws/vpcflow/manifest.yml | 2 ++ x-pack/filebeat/modules.d/aws.yml.disabled | 36 +++++++++++++++++++ 15 files changed, 144 insertions(+) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 4973a0ae915..a7c8f2e3090 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -142,6 +142,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + cloudwatch: enabled: false @@ -176,6 +182,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + ec2: enabled: false @@ -210,6 +222,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + elb: enabled: false @@ -244,6 +262,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + s3access: enabled: false @@ -278,6 +302,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + vpcflow: enabled: false @@ -312,6 +342,12 @@ filebeat.modules: # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + #-------------------------------- Azure Module -------------------------------- - module: azure # All logs diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index b7e0c25b674..e4b521e467f 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -45,6 +45,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + cloudwatch: enabled: false @@ -79,6 +85,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + ec2: enabled: false @@ -113,6 +125,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + elb: enabled: false @@ -147,6 +165,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + s3access: enabled: false @@ -181,6 +205,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + vpcflow: enabled: false @@ -214,3 +244,9 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 50c1006d32a..0f099f3f3fe 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -55,6 +55,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index 732967ff0b0..03c7acf1336 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -21,6 +21,8 @@ var: default: true - name: process_insight_logs default: true + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml index 1b0c132edcd..7bc16026f18 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml @@ -41,6 +41,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index 2878c79936d..5d9931b2e40 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -15,6 +15,8 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/s3.yml index 1b0c132edcd..7bc16026f18 100644 --- a/x-pack/filebeat/module/aws/ec2/config/s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/s3.yml @@ -41,6 +41,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml index 2878c79936d..5d9931b2e40 100644 --- a/x-pack/filebeat/module/aws/ec2/manifest.yml +++ b/x-pack/filebeat/module/aws/ec2/manifest.yml @@ -15,6 +15,8 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml index 1b0c132edcd..7bc16026f18 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/s3.yml @@ -41,6 +41,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index f823ccbacce..dc95f6abb7e 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -15,6 +15,8 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml index 1b0c132edcd..7bc16026f18 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/s3.yml @@ -41,6 +41,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index 2878c79936d..5d9931b2e40 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -15,6 +15,8 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 20ae31ccbdf..32b1a363024 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -43,6 +43,10 @@ role_arn: {{ .role_arn }} fips_enabled: {{ .fips_enabled }} {{ end }} +{{ if .max_number_of_messages }} +max_number_of_messages: {{ .max_number_of_messages }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index c7df14a4050..19f40c7a3f7 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -15,6 +15,8 @@ var: - name: role_arn - name: tags default: [forwarded] + - name: fips_enabled + - name: max_number_of_messages ingest_pipeline: ingest/pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index d619d200d8a..904bd976ce6 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -48,6 +48,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + cloudwatch: enabled: false @@ -82,6 +88,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + ec2: enabled: false @@ -116,6 +128,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + elb: enabled: false @@ -150,6 +168,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + s3access: enabled: false @@ -184,6 +208,12 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5 + vpcflow: enabled: false @@ -217,3 +247,9 @@ # AWS IAM Role to assume #var.role_arn: arn:aws:iam::123456789012:role/test-mb + + # Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. + #var.fips_enabled: false + + # The maximum number of messages to return from SQS. Valid values: 1 to 10. + #var.max_number_of_messages: 5