Add text & flattened fields in aws cloudtrail fileset (#19121) (#19809)

AWS cloudtrail events have the following fields where the subfields
are highley variable: requestParameters, responseElements,
additionalEventData and serviceEventDetails.

multi_fields added to following fields
- aws.cloudtrail.request_parameters
- aws.cloudtrail.response_elements
- aws.cloudtrail.additiona_eventdata
- aws.cloudtrail.service_event_details

flattened version of the fields are stored here:
- aws.cloudtrail.flattened.request_parameters
- aws.cloudtrail.flattened.response_elements
- aws.cloudtrail.flattened.additiona_eventdata
- aws.cloudtrail.flattened.service_event_details

Closes #18866

(cherry picked from commit d16ecc9)

CHANGELOG.next.asciidoc

@@ -514,6 +514,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]
 - Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376]
 - Add automatic retries and exponential backoff to httpjson input. {pull}18956[18956]
+- Add text & flattened versions of fields with unknown subfields in aws cloudtrail fileset. {issue}18866[18866] {pull}19121[19121]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -1228,6 +1228,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.request_parameters.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.response_elements`*::
 +
 --
@@ -1237,6 +1244,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.response_elements.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.additional_eventdata`*::
 +
 --
@@ -1246,6 +1260,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.additional_eventdata.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.request_id`*::
 +
 --
@@ -1342,6 +1363,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.service_event_details.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.shared_event_id`*::
 +
 --
@@ -1400,6 +1428,49 @@ type: boolean
 
 --
 
+[float]
+=== flattened
+
+ES flattened datatype for objects where the subfields aren't known in advance.
+
+
+*`aws.cloudtrail.flattened.additional_eventdata`*::
++
+--
+Additional data about the event that was not part of the request or response.
+
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.request_parameters`*::
++
+--
+The parameters, if any, that were sent with the request.
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.response_elements`*::
++
+--
+The response element for actions that make changes (create, update, or delete actions).
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.service_event_details`*::
++
+--
+Identifies the service event, including what triggered the event and the result.
+
+type: flattened
+
+--
+
 [float]
 === cloudwatch
 

libbeat/mapping/field.go

@@ -138,7 +138,7 @@ func (f *Field) validateType() error {
 		if f.Format != "" {
 			return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format)
 		}
-	case "object", "group", "nested":
+	case "object", "group", "nested", "flattened":
 		// No check for them yet
 	case "":
 		// Module keys, not used as fields

libbeat/tests/system/beat/beat.py

@@ -571,7 +571,7 @@ def extract_fields(doc_list, name):
                     aliases.extend(subaliases)
                 else:
                     fields.append(newName)
-                    if field.get("type") in ["object", "geo_point"]:
+                    if field.get("type") in ["object", "geo_point", "flattened"]:
                         dictfields.append(newName)
 
                 if field.get("type") == "object" and field.get("object_type") == "histogram":

x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml

@@ -90,16 +90,28 @@
       type: keyword
       description: >-
         The parameters, if any, that were sent with the request.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: response_elements
       type: keyword
       description: >-
         The response element for actions that make changes (create,
         update, or delete actions).
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: additional_eventdata
       type: keyword
       description: >-
         Additional data about the event that was not part of the
         request or response.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: request_id
       type: keyword
       description: >-
@@ -149,6 +161,10 @@
       description: >-
         Identifies the service event, including what triggered the
         event and the result.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: shared_event_id
       type: keyword
       description: >-
@@ -183,3 +199,27 @@
               description: >-
                 Identifies whether multi factor authentication was
                 used during ConsoleLogin
+    - name: flattened
+      type: group
+      description: >-
+        ES flattened datatype for objects where the subfields aren't known in advance.
+      fields:
+        - name: additional_eventdata
+          type: flattened
+          description: >
+            Additional data about the event that was not part of the
+            request or response.
+        - name: request_parameters
+          type: flattened
+          description: >-
+            The parameters, if any, that were sent with the request.
+        - name: response_elements
+          type: flattened
+          description: >-
+            The response element for actions that make changes (create,
+            update, or delete actions).
+        - name: service_event_details
+          type: flattened
+          description: >-
+            Identifies the service event, including what triggered the
+            event and the result.

x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml

@@ -135,25 +135,37 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
+  - rename:
+      field: json.requestParameters
+      target_field: "aws.cloudtrail.flattened.request_parameters"
+      if: ctx.json.requestParameters != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.requestParameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
+        if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
+          ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
         }
       ignore_failure: true
+  - rename:
+      field: json.responseElements
+      target_field: "aws.cloudtrail.flattened.response_elements"
+      if: ctx.json.responseElements != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.responseElements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
+        if (ctx.aws.cloudtrail.flattened.response_elements != null) {
+          ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
         }
       ignore_failure: true
+  - rename:
+      field: json.additionalEventData
+      target_field: "aws.cloudtrail.flattened.additional_eventdata"
+      if: ctx?.json?.additionalEventData != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.additionalEventData != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
+        if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
+          ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
         }
       ignore_failure: true
   - rename:
@@ -196,11 +208,15 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
+  - rename:
+      field: json.serviceEventDetails
+      target_field: "aws.cloudtrail.flattened.service_event_details"
+      if: ctx.json.serviceEventDetails != null
   - script:
       lang: painless
       source: |
-        if (ctx.json.serviceEventDetails != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
+        if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
+          ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
         }
       ignore_failure: true
   - rename:
@@ -226,14 +242,11 @@ processors:
           }
           ctx.related.user.add(userName);
         }
-
-
-
-        if (ctx.json?.requestParameters.userName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.userName);
+        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
         }
-        if (ctx.json?.requestParameters.newUserName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.newUserName);
+        if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
         }
 
   - script:
@@ -244,18 +257,18 @@ processors:
           return;
         }
         Map aed_map = new HashMap();
-        if (ctx.json?.additionalEventData?.MobileVersion != null) {
-          if (ctx.json.additionalEventData.MobileVersion == 'No') {
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) {
+          if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') {
             aed_map.put("mobile_version", false);
           } else {
             aed_map.put("mobile_version", true);
           }
         }
-        if (ctx.json?.additionalEventData?.LoginTo != null) {
-          aed_map.put("login_to", ctx.json.additionalEventData.LoginTo);
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) {
+          aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo);
         }
-        if (ctx.json?.additionalEventData?.MFAUsed != null) {
-          if (ctx.json.additionalEventData.MFAUsed == 'No') {
+        if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) {
+          if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') {
             aed_map.put("mfa_used", false);
           } else {
             aed_map.put("mfa_used", true);
@@ -598,8 +611,8 @@ processors:
             return;
         }
 
-        if (ctx.event.action == 'ConsoleLogin' && ctx.json?.responseElements.ConsoleLogin != null) {
-            ctx.event.outcome = Processors.lowercase(ctx.json.responseElements.ConsoleLogin);
+        if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) {
+            ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin);
         }
         
         def hm = new HashMap(params.get(ctx.event.action));

x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json

@@ -2,6 +2,8 @@
     {
         "@timestamp": "2014-03-25T21:08:14.000Z",
         "aws.cloudtrail.event_version": "1.0",
+        "aws.cloudtrail.flattened.request_parameters.groupName": "admin",
+        "aws.cloudtrail.flattened.request_parameters.userName": "Bob",
         "aws.cloudtrail.request_parameters": "{groupName=admin, userName=Bob}",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",

x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json

@@ -3,6 +3,29 @@
         "@timestamp": "2019-10-02T22:12:29.000Z",
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
+        "aws.cloudtrail.flattened.request_parameters.durationSeconds": 3600,
+        "aws.cloudtrail.flattened.request_parameters.incomingTransitiveTags.Department": "Engineering",
+        "aws.cloudtrail.flattened.request_parameters.roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
+        "aws.cloudtrail.flattened.request_parameters.roleSessionName": "Role2WithTags",
+        "aws.cloudtrail.flattened.request_parameters.tags": [
+            {
+                "key": "Email",
+                "value": "johndoe@example.com"
+            },
+            {
+                "key": "CostCenter",
+                "value": "12345"
+            }
+        ],
+        "aws.cloudtrail.flattened.request_parameters.transitiveTagKeys": [
+            "Email",
+            "CostCenter"
+        ],
+        "aws.cloudtrail.flattened.response_elements.assumedRoleUser.arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
+        "aws.cloudtrail.flattened.response_elements.assumedRoleUser.assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
+        "aws.cloudtrail.flattened.response_elements.credentials.accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
+        "aws.cloudtrail.flattened.response_elements.credentials.expiration": "Oct 2, 2019 11:12:29 PM",
+        "aws.cloudtrail.flattened.response_elements.credentials.sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
         "aws.cloudtrail.recipient_account_id": "111111111111",
         "aws.cloudtrail.request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}",
         "aws.cloudtrail.response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}",

x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json

@@ -6,6 +6,10 @@
         "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.event_version": "1.05",
+        "aws.cloudtrail.flattened.additional_eventdata.LoginTo": "https://console.aws.amazon.com/s3/",
+        "aws.cloudtrail.flattened.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.flattened.additional_eventdata.MobileVersion": "No",
+        "aws.cloudtrail.flattened.response_elements.ConsoleLogin": "Success",
         "aws.cloudtrail.response_elements": "{ConsoleLogin=Success}",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JohnDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
@@ -52,6 +56,10 @@
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
+        "aws.cloudtrail.flattened.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.flattened.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.flattened.additional_eventdata.MobileVersion": "No",
+        "aws.cloudtrail.flattened.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JaneDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
@@ -98,6 +106,10 @@
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
+        "aws.cloudtrail.flattened.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.flattened.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.flattened.additional_eventdata.MobileVersion": "No",
+        "aws.cloudtrail.flattened.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
         "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",