Filebeat: Update aws/cloudtrail dataset to ECS 1.8

CHANGELOG.next.asciidoc

@@ -833,6 +833,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
 - Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
+- Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
 
 *Heartbeat*
 

x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml

@@ -66,4 +66,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0

x-pack/filebeat/module/aws/cloudtrail/config/file.yml

@@ -11,4 +11,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -27,6 +27,11 @@ processors:
       field: "json.userIdentity.type"
       target_field: "aws.cloudtrail.user_identity.type"
       ignore_failure: true
+  - append:
+      field: related.user
+      value: '{{json.userIdentity.userName}}'
+      allow_duplicates: false
+      if: 'ctx.json?.userIdentity?.userName != null'
   - rename:
       field: "json.userIdentity.userName"
       target_field: "user.name"
@@ -225,28 +230,16 @@ processors:
       field: "json.vpcEndpointId"
       target_field: "aws.cloudtrail.vpc_endpoint_id"
       ignore_failure: true
-  - script:
-      lang: painless
-      ignore_failure: true
-      source: >-
-        void addRelatedUser(def ctx, String userName) {
-          if (ctx.related == null) {
-            Map map = new HashMap();
-            ctx.put("related", map);
-          }
-          if (ctx.related.user == null) {
-            ArrayList al = new ArrayList();
-            ctx.related.put("user", al);
-          }
-          ctx.related.user.add(userName);
-        }
-        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
-          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
-        }
-        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
-          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
-        }
-
+  - append:
+      field: related.user
+      value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
+      allow_duplicates: false
+      if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.userName != null'
+  - append:
+      field: related.user
+      value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
+      allow_duplicates: false
+      if: 'ctx.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null'
   - script:
       lang: painless
       ignore_failure: true
@@ -685,6 +678,32 @@ processors:
       field: "json.insightDetails"
       target_field: "aws.cloudtrail.insight_details"
       ignore_failure: true
+  - set:
+      field: group.id
+      value: '{{aws.cloudtrail.flattened.response_elements.group.groupId}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.target.id
+      value: '{{aws.cloudtrail.flattened.response_elements.user.userId}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.changes.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.newUserName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: group.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.groupName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+  - set:
+      field: user.target.name
+      value: '{{aws.cloudtrail.flattened.request_parameters.userName}}'
+      ignore_empty_value: true
+      ignore_failure: true
+
   - remove:
       field:
         - "json"

x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json

@@ -27,9 +27,11 @@
             "change"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "admin",
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -40,6 +42,7 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "AWSConsole"

x-pack/filebeat/module/aws/cloudtrail/test/change-password-json.log-expected.json

@@ -29,6 +29,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -70,6 +73,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 720,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json

@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "JohnDoe"
+        ],
         "service.type": "aws",
         "source.address": "192.0.2.110",
         "source.ip": "192.0.2.110",
@@ -82,6 +85,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 658,
+        "related.user": [
+            "JaneDoe"
+        ],
         "service.type": "aws",
         "source.address": "192.0.2.100",
         "source.ip": "192.0.2.100",

x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json

@@ -38,6 +38,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -48,6 +49,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json

@@ -36,8 +36,13 @@
             "creation"
         ],
         "fileset.name": "cloudtrail",
+        "group.id": "EXAMPLE_ID",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -80,8 +85,12 @@
             "creation"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 903,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json

@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "72.21.198.64",
         "source.as.number": 16509,

x-pack/filebeat/module/aws/cloudtrail/test/create-trail-json.log-expected.json

@@ -41,6 +41,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/create-user-json.log-expected.json

@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,8 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.id": "EXAMPLEUSERID",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "aws-cli",
         "user_agent.original": "aws-cli/1.3.2 Python/2.7.5 Windows/7",

x-pack/filebeat/module/aws/cloudtrail/test/create-virtual-mfa-device-json.log-expected.json

@@ -34,6 +34,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/deactivate-mfa-device-json.log-expected.json

@@ -44,6 +44,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Alice",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

x-pack/filebeat/module/aws/cloudtrail/test/delete-access-key-json.log-expected.json

@@ -34,6 +34,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -44,6 +45,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

x-pack/filebeat/module/aws/cloudtrail/test/delete-group-json.log-expected.json

@@ -30,8 +30,12 @@
             "deletion"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",
@@ -74,8 +78,12 @@
             "deletion"
         ],
         "fileset.name": "cloudtrail",
+        "group.name": "TEST-GROUP",
         "input.type": "log",
         "log.offset": 747,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/delete-ssh-public-key-json.log-expected.json

@@ -34,6 +34,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -44,6 +45,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

x-pack/filebeat/module/aws/cloudtrail/test/delete-trail-json.log-expected.json

@@ -24,6 +24,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/delete-user-json.log-expected.json

@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,7 @@
         ],
         "user.id": "EX_PRINCIPAL_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "signin.amazonaws.com"

x-pack/filebeat/module/aws/cloudtrail/test/delete-virtual-mfa-device-json.log-expected.json

@@ -32,6 +32,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "Alice"
+        ],
         "service.type": "aws",
         "source.address": "127.0.0.1",
         "source.ip": "127.0.0.1",

x-pack/filebeat/module/aws/cloudtrail/test/describe_configuration_recorders-json.log-expected.json

@@ -25,6 +25,9 @@
         "fileset.name": "cloudtrail",
         "input.type": "log",
         "log.offset": 0,
+        "related.user": [
+            "REDACTED"
+        ],
         "service.type": "aws",
         "source.address": "REDACTED",
         "tags": [

x-pack/filebeat/module/aws/cloudtrail/test/enable-mfa-device-json.log-expected.json

@@ -33,6 +33,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.user": [
+            "Alice",
             "Bob"
         ],
         "service.type": "aws",
@@ -43,6 +44,7 @@
         ],
         "user.id": "EXAMPLE_ID",
         "user.name": "Alice",
+        "user.target.name": "Bob",
         "user_agent.device.name": "Other",
         "user_agent.name": "Other",
         "user_agent.original": "console.amazonaws.com"