Cherry-pick #15656 to 7.x: [Filebeat] Add support for specifying AWS cred file

CHANGELOG.next.asciidoc

@@ -78,6 +78,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449]
 - netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449]
 - Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553]
+- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
 
 *Heartbeat*
 

filebeat/docs/modules/aws.asciidoc

@@ -45,41 +45,69 @@ Example config:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   elb:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   vpcflow:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-      #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   cloudtrail:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
 
 AWS SQS queue url.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
 *`var.credential_profile_name`*::
 
 AWS credential profile name.

x-pack/filebeat/filebeat.reference.yml

@@ -102,7 +102,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -111,7 +117,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   vpcflow:
@@ -120,7 +132,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   cloudtrail:
@@ -129,7 +147,13 @@ filebeat.modules:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
 #-------------------------------- Azure Module --------------------------------

x-pack/filebeat/module/aws/_meta/config.yml

@@ -5,7 +5,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   elb:
@@ -14,7 +20,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   vpcflow:
@@ -23,7 +35,13 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
   cloudtrail:
@@ -32,5 +50,11 @@
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
+    # If not set the default profile is used
     #var.credential_profile_name: fb-aws

x-pack/filebeat/module/aws/_meta/docs.asciidoc

@@ -40,41 +40,69 @@ Example config:
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   elb:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   vpcflow:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-      #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 
   cloudtrail:
     enabled: false
 
     # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
 
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    # var.shared_credential_file: /etc/filebeat/aws_credentials
+
     # Profile name for aws credential
-    #var.credential_profile_name: fb-aws
+    # If not set the default profile is used
+    # var.credential_profile_name: fb-aws
 ----
 
 *`var.queue_url`*::
 
 AWS SQS queue url.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
 *`var.credential_profile_name`*::
 
 AWS credential profile name.

x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml

@@ -2,9 +2,17 @@
 
 type: s3
 queue_url: {{ .queue_url }}
-credential_profile_name: {{ .credential_profile_name }}
 expand_event_list_from_field: Records
 
+{{ if .credential_profile_name }}
+credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
+
+
 {{ else if eq .input "file" }}
 
 type: log

x-pack/filebeat/module/aws/cloudtrail/manifest.yml

@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/cloudtrail.yml

x-pack/filebeat/module/aws/elb/config/s3.yml

@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}

x-pack/filebeat/module/aws/elb/manifest.yml

@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

x-pack/filebeat/module/aws/s3access/config/s3.yml

@@ -1,3 +1,10 @@
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}

x-pack/filebeat/module/aws/s3access/manifest.yml

@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

x-pack/filebeat/module/aws/vpcflow/config/input.yml

@@ -2,7 +2,14 @@
 
 type: s3
 queue_url: {{ .queue_url }}
+
+{{ if .credential_profile_name }}
 credential_profile_name: {{ .credential_profile_name }}
+{{ end }}
+
+{{ if .shared_credential_file }}
+shared_credential_file: {{ .shared_credential_file }}
+{{ end }}
 
 {{ else if eq .input "file" }}
 

x-pack/filebeat/module/aws/vpcflow/manifest.yml

@@ -3,6 +3,8 @@ module_version: 1.0
 var:
   - name: input
     default: s3
+  - name: shared_credential_file
+  - name: credential_profile_name
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml