change field types in aws cloudtrail fileset

A string representation of the following fields was stored. Changing to an object which is more easily searched. - request_parameters - response_elements - additional_eventdata - service_event_details Closes elastic#18866
leehinman · Jun 2, 2020 · 793bdc5 · 793bdc5
1 parent 60c6ba4
commit 793bdc5
Show file tree

Hide file tree

Showing 34 changed files with 180 additions and 99 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -41,6 +41,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 * iptables {pull}18756[18756]
 * Checkpoint {pull}18754[18754]
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
+- In aws cloudtrail fileset change type of field from string to object for request_parameters, response_elements, additional_eventdata & service_event_details. {issue}18866[18866] {pull}18909[18909]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1223,7 +1223,7 @@ type: keyword
 --
 The parameters, if any, that were sent with the request.
 
-type: keyword
+type: object
 
 --
 
@@ -1232,7 +1232,7 @@ type: keyword
 --
 The response element for actions that make changes (create, update, or delete actions).
 
-type: keyword
+type: object
 
 --
 
@@ -1241,7 +1241,7 @@ type: keyword
 --
 Additional data about the event that was not part of the request or response.
 
-type: keyword
+type: object
 
 --
 
@@ -1337,7 +1337,7 @@ type: keyword
 --
 Identifies the service event, including what triggered the event and the result.
 
-type: keyword
+type: object
 
 --
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
@@ -87,16 +87,16 @@
       description: >-
         If the request returns an error, the description of the error.
     - name: request_parameters
-      type: keyword
+      type: object
       description: >-
         The parameters, if any, that were sent with the request.
     - name: response_elements
-      type: keyword
+      type: object
       description: >-
         The response element for actions that make changes (create,
         update, or delete actions).
     - name: additional_eventdata
-      type: keyword
+      type: object
       description: >-
         Additional data about the event that was not part of the
         request or response.
@@ -145,7 +145,7 @@
       description: >-
         Represents the account ID that received this event.
     - name: service_event_details
-      type: keyword
+      type: object
       description: >-
         Identifies the service event, including what triggered the
         event and the result.

diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
@@ -118,27 +118,18 @@ processors:
       field: "json.errorMessage"
       target_field: "aws.cloudtrail.error_message"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.requestParameters != null) {
-          ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
-        }
-      ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.responseElements != null) {
-          ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
-        }
-      ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.additionalEventData != null) {
-          ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
-        }
-      ignore_failure: true
+  - rename:
+      field: "json.requestParameters"
+      target_field: "aws.cloudtrail.request_parameters"
+      if: "ctx.json.requestParameters != null"
+  - rename:
+      field: "json.responseElements"
+      target_field: "aws.cloudtrail.response_elements"
+      if: "ctx.json.responseElements != null"
+  - rename:
+      field: "json.additionalEventData"
+      target_field: "aws.cloudtrail.additional_eventdata"
+      if: "ctx.json.additionalEventData != null"
   - rename:
       field: "json.requestId"
       target_field: "aws.cloudtrail.request_id"
@@ -179,13 +170,10 @@ processors:
       field: "json.recipientAccountId"
       target_field: "aws.cloudtrail.recipient_account_id"
       ignore_failure: true
-  - script:
-      lang: painless
-      source: |
-        if (ctx.json.serviceEventDetails != null) {
-          ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
-        }
-      ignore_failure: true
+  - rename:
+      field: "json.serviceEventDetails"
+      target_field: "aws.cloudtrail.service_event_details"
+      if: "ctx.json.serviceEventDetails != null"
   - rename:
       field: "json.sharedEventId"
       target_field: "aws.cloudtrail.shared_event_id"
@@ -218,40 +206,40 @@ processors:
             ctx.event.outcome = 'success'
         }
 
-        if (ctx.json?.eventName == 'ConsoleLogin') {
+        if (ctx?.event?.action == 'ConsoleLogin') {
           ctx.event.category = 'authentication';
-          if (ctx.json?.responseElements.ConsoleLogin != null) {
-            ctx.event.outcome = Processors.lowercase(ctx.json.responseElements.ConsoleLogin);
+          if (ctx?.aws?.cloudtrail?.response_elements?.ConsoleLogin != null) {
+            ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.response_elements.ConsoleLogin);
           }
         }
 
-        if (ctx.json?.requestParameters.userName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.userName);
+        if (ctx?.aws?.cloudtrail?.request_parameters.userName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.request_parameters.userName);
         }
-        if (ctx.json?.requestParameters.newUserName != null) {
-          addRelatedUser(ctx, ctx.json.requestParameters.newUserName);
+        if (ctx?.aws?.cloudtrail?.request_parameters.newUserName != null) {
+          addRelatedUser(ctx, ctx.aws.cloudtrail.request_parameters.newUserName);
         }
 
   - script:
       lang: painless
       ignore_failure: true
       source: >-
-        if (ctx.json?.eventName != 'ConsoleLogin') {
+        if (ctx?.event?.action != 'ConsoleLogin') {
           return;
         }
         Map aed_map = new HashMap();
-        if (ctx.json?.additionalEventData?.MobileVersion != null) {
-          if (ctx.json.additionalEventData.MobileVersion == 'No') {
+        if (ctx?.aws?.cloudtrail?.additional_eventdata?.MobileVersion != null) {
+          if (ctx.aws.cloudtrail.additional_eventdata.MobileVersion == 'No') {
             aed_map.put("mobile_version", false);
           } else {
             aed_map.put("mobile_version", true);
           }
         }
-        if (ctx.json?.additionalEventData?.LoginTo != null) {
-          aed_map.put("login_to", ctx.json.additionalEventData.LoginTo);
+        if (ctx?.aws?.cloudtrail?.additional_eventdata?.LoginTo != null) {
+          aed_map.put("login_to", ctx.aws.cloudtrail.additional_eventdata.LoginTo);
         }
-        if (ctx.json?.additionalEventData?.MFAUsed != null) {
-          if (ctx.json.additionalEventData.MFAUsed == 'No') {
+        if (ctx?.aws?.cloudtrail?.additional_eventdata?.MFAUsed != null) {
+          if (ctx.aws.cloudtrail.additional_eventdata.MFAUsed == 'No') {
             aed_map.put("mfa_used", false);
           } else {
             aed_map.put("mfa_used", true);

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/add-user-to-group-json.log-expected.json
@@ -2,7 +2,8 @@
     {
         "@timestamp": "2014-03-25T21:08:14.000Z",
         "aws.cloudtrail.event_version": "1.0",
-        "aws.cloudtrail.request_parameters": "{groupName=admin, userName=Bob}",
+        "aws.cloudtrail.request_parameters.groupName": "admin",
+        "aws.cloudtrail.request_parameters.userName": "Bob",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-25T18:45:11.000Z",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/assume-role-json.log-expected.json
@@ -4,8 +4,29 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "111111111111",
-        "aws.cloudtrail.request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}",
-        "aws.cloudtrail.response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}",
+        "aws.cloudtrail.request_parameters.durationSeconds": 3600,
+        "aws.cloudtrail.request_parameters.incomingTransitiveTags.Department": "Engineering",
+        "aws.cloudtrail.request_parameters.roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
+        "aws.cloudtrail.request_parameters.roleSessionName": "Role2WithTags",
+        "aws.cloudtrail.request_parameters.tags": [
+            {
+                "key": "Email",
+                "value": "johndoe@example.com"
+            },
+            {
+                "key": "CostCenter",
+                "value": "12345"
+            }
+        ],
+        "aws.cloudtrail.request_parameters.transitiveTagKeys": [
+            "Email",
+            "CostCenter"
+        ],
+        "aws.cloudtrail.response_elements.assumedRoleUser.arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
+        "aws.cloudtrail.response_elements.assumedRoleUser.assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
+        "aws.cloudtrail.response_elements.credentials.accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
+        "aws.cloudtrail.response_elements.credentials.expiration": "Oct 2, 2019 11:12:29 PM",
+        "aws.cloudtrail.response_elements.credentials.sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
         "aws.cloudtrail.user_identity.access_key_id": "AKIAI44QH8DHBEXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2019-10-02T21:50:54.000Z",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/console-login-json.log-expected.json
@@ -1,12 +1,14 @@
 [
     {
         "@timestamp": "2014-07-16T15:49:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/s3/, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/s3/",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/s3/",
         "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Success}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Success",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JohnDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
         "cloud.account.id": "111122223333",
@@ -42,13 +44,15 @@
     },
     {
         "@timestamp": "2014-07-08T17:35:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
         "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JaneDoe",
         "aws.cloudtrail.user_identity.type": "IAMUser",
         "cloud.account.id": "111122223333",
@@ -84,13 +88,15 @@
     },
     {
         "@timestamp": "2014-07-08T17:35:27.000Z",
-        "aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
+        "aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
+        "aws.cloudtrail.additional_eventdata.MFAUsed": "No",
+        "aws.cloudtrail.additional_eventdata.MobileVersion": "No",
         "aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
         "aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
         "aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
         "aws.cloudtrail.error_message": "Failed authentication",
         "aws.cloudtrail.event_version": "1.05",
-        "aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
+        "aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
         "aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
         "aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
         "aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-access-key-json.log-expected.json
@@ -4,8 +4,11 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{userName=Bob}",
-        "aws.cloudtrail.response_elements": "{accessKey={accessKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Active, createDate=Jan 8, 2020 8:43:06 PM}}",
+        "aws.cloudtrail.request_parameters.userName": "Bob",
+        "aws.cloudtrail.response_elements.accessKey.accessKeyId": "EXAMPLE_KEY_ID",
+        "aws.cloudtrail.response_elements.accessKey.createDate": "Jan 8, 2020 8:43:06 PM",
+        "aws.cloudtrail.response_elements.accessKey.status": "Active",
+        "aws.cloudtrail.response_elements.accessKey.userName": "Bob",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-group-json.log-expected.json
@@ -4,8 +4,12 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
-        "aws.cloudtrail.response_elements": "{group={path=/, groupName=TEST-GROUP, groupId=EXAMPLE_ID, arn=arn:aws:iam::0123456789012:group/TEST-GROUP, createDate=Jan 9, 2020 1:48:44 AM}}",
+        "aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.arn": "arn:aws:iam::0123456789012:group/TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.createDate": "Jan 9, 2020 1:48:44 AM",
+        "aws.cloudtrail.response_elements.group.groupId": "EXAMPLE_ID",
+        "aws.cloudtrail.response_elements.group.groupName": "TEST-GROUP",
+        "aws.cloudtrail.response_elements.group.path": "/",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
@@ -45,7 +49,7 @@
         "aws.cloudtrail.event_type": "AwsApiCall",
         "aws.cloudtrail.event_version": "1.05",
         "aws.cloudtrail.recipient_account_id": "0123456789012",
-        "aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
+        "aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
         "aws.cloudtrail.user_identity.type": "IAMUser",

diff --git a/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json b/x-pack/filebeat/module/aws/cloudtrail/test/create-key-pair-json.log-expected.json
@@ -2,8 +2,10 @@
     {
         "@timestamp": "2014-03-06T17:10:34.000Z",
         "aws.cloudtrail.event_version": "1.0",
-        "aws.cloudtrail.request_parameters": "{keyName=mykeypair}",
-        "aws.cloudtrail.response_elements": "{keyMaterial=<sensitiveDataRemoved>, keyFingerprint=30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21, keyName=mykeypair}",
+        "aws.cloudtrail.request_parameters.keyName": "mykeypair",
+        "aws.cloudtrail.response_elements.keyFingerprint": "30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21",
+        "aws.cloudtrail.response_elements.keyMaterial": "<sensitiveDataRemoved>",
+        "aws.cloudtrail.response_elements.keyName": "mykeypair",
         "aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
         "aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
         "aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-06T15:15:06.000Z",