Skip to content

Commit

Permalink
Change field type in aws cloudtrail fileset to flattened
Browse files Browse the repository at this point in the history
A string representation of the following fields was stored and indexed
as keyword.  Changing to a flattened object which is more easily
searched.

- request_parameters
- response_elements
- additional_eventdata
- service_event_details

Closes elastic#18866
  • Loading branch information
leehinman committed Jun 5, 2020
1 parent 0a669eb commit f004955
Show file tree
Hide file tree
Showing 36 changed files with 168 additions and 138 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
* iptables {pull}18756[18756]
* Checkpoint {pull}18754[18754]
- Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
- In aws cloudtrail fileset change type of field from keyword to flattened for request_parameters, response_elements, additional_eventdata & service_event_details. {issue}18866[18866] {pull}19020[19020]

*Heartbeat*

Expand Down
8 changes: 4 additions & 4 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -1223,7 +1223,7 @@ type: keyword
--
The parameters, if any, that were sent with the request.
type: keyword
type: flattened
--
Expand All @@ -1232,7 +1232,7 @@ type: keyword
--
The response element for actions that make changes (create, update, or delete actions).
type: keyword
type: flattened
--
Expand All @@ -1241,7 +1241,7 @@ type: keyword
--
Additional data about the event that was not part of the request or response.
type: keyword
type: flattened
--
Expand Down Expand Up @@ -1337,7 +1337,7 @@ type: keyword
--
Identifies the service event, including what triggered the event and the result.
type: keyword
type: flattened
--
Expand Down
2 changes: 1 addition & 1 deletion libbeat/mapping/field.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (f *Field) validateType() error {
if f.Format != "" {
return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format)
}
case "object", "group", "nested":
case "object", "group", "nested", "flattened":
// No check for them yet
case "":
// Module keys, not used as fields
Expand Down
2 changes: 1 addition & 1 deletion libbeat/tests/system/beat/beat.py
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,7 @@ def extract_fields(doc_list, name):
aliases.extend(subaliases)
else:
fields.append(newName)
if field.get("type") in ["object", "geo_point"]:
if field.get("type") in ["object", "geo_point", "flattened"]:
dictfields.append(newName)

if field.get("type") == "object" and field.get("object_type") == "histogram":
Expand Down
8 changes: 4 additions & 4 deletions x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,16 +87,16 @@
description: >-
If the request returns an error, the description of the error.
- name: request_parameters
type: keyword
type: flattened
description: >-
The parameters, if any, that were sent with the request.
- name: response_elements
type: keyword
type: flattened
description: >-
The response element for actions that make changes (create,
update, or delete actions).
- name: additional_eventdata
type: keyword
type: flattened
description: >-
Additional data about the event that was not part of the
request or response.
Expand Down Expand Up @@ -145,7 +145,7 @@
description: >-
Represents the account ID that received this event.
- name: service_event_details
type: keyword
type: flattened
description: >-
Identifies the service event, including what triggered the
event and the result.
Expand Down
40 changes: 16 additions & 24 deletions x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,26 +118,20 @@ processors:
field: "json.errorMessage"
target_field: "aws.cloudtrail.error_message"
ignore_failure: true
- script:
lang: painless
source: |
if (ctx.json.requestParameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
}
- rename:
field: "json.requestParameters"
target_field: "aws.cloudtrail.request_parameters"
if: "ctx.json.requestParameters != null"
ignore_failure: true
- script:
lang: painless
source: |
if (ctx.json.responseElements != null) {
ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
}
- rename:
field: "json.responseElements"
target_field: "aws.cloudtrail.response_elements"
if: "ctx.json.responseElements != null"
ignore_failure: true
- script:
lang: painless
source: |
if (ctx.json.additionalEventData != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
}
- rename:
field: "json.additionalEventData"
target_field: "aws.cloudtrail.additional_eventdata"
if: "ctx.json.additionalEventData != null"
ignore_failure: true
- rename:
field: "json.requestId"
Expand Down Expand Up @@ -179,12 +173,10 @@ processors:
field: "json.recipientAccountId"
target_field: "aws.cloudtrail.recipient_account_id"
ignore_failure: true
- script:
lang: painless
source: |
if (ctx.json.serviceEventDetails != null) {
ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
}
- rename:
field: "json.serviceEventDetails"
target_field: "aws.cloudtrail.service_event_details"
if: "ctx.json.serviceEventDetails != null"
ignore_failure: true
- rename:
field: "json.sharedEventId"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
{
"@timestamp": "2014-03-25T21:08:14.000Z",
"aws.cloudtrail.event_version": "1.0",
"aws.cloudtrail.request_parameters": "{groupName=admin, userName=Bob}",
"aws.cloudtrail.request_parameters.groupName": "admin",
"aws.cloudtrail.request_parameters.userName": "Bob",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
"aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-25T18:45:11.000Z",
Expand All @@ -21,9 +22,6 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Bob"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,29 @@
"aws.cloudtrail.event_type": "AwsApiCall",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.recipient_account_id": "111111111111",
"aws.cloudtrail.request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}",
"aws.cloudtrail.response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}",
"aws.cloudtrail.request_parameters.durationSeconds": 3600,
"aws.cloudtrail.request_parameters.incomingTransitiveTags.Department": "Engineering",
"aws.cloudtrail.request_parameters.roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
"aws.cloudtrail.request_parameters.roleSessionName": "Role2WithTags",
"aws.cloudtrail.request_parameters.tags": [
{
"key": "Email",
"value": "johndoe@example.com"
},
{
"key": "CostCenter",
"value": "12345"
}
],
"aws.cloudtrail.request_parameters.transitiveTagKeys": [
"Email",
"CostCenter"
],
"aws.cloudtrail.response_elements.assumedRoleUser.arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags",
"aws.cloudtrail.response_elements.assumedRoleUser.assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
"aws.cloudtrail.response_elements.credentials.accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
"aws.cloudtrail.response_elements.credentials.expiration": "Oct 2, 2019 11:12:29 PM",
"aws.cloudtrail.response_elements.credentials.sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
"aws.cloudtrail.user_identity.access_key_id": "AKIAI44QH8DHBEXAMPLE",
"aws.cloudtrail.user_identity.arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1",
"aws.cloudtrail.user_identity.session_context.creation_date": "2019-10-02T21:50:54.000Z",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
[
{
"@timestamp": "2014-07-16T15:49:27.000Z",
"aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/s3/, MobileVersion=No, MFAUsed=No}",
"aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/s3/",
"aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
"aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
"aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/s3/",
"aws.cloudtrail.additional_eventdata.MFAUsed": "No",
"aws.cloudtrail.additional_eventdata.MobileVersion": "No",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.response_elements": "{ConsoleLogin=Success}",
"aws.cloudtrail.response_elements.ConsoleLogin": "Success",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JohnDoe",
"aws.cloudtrail.user_identity.type": "IAMUser",
"cloud.account.id": "111122223333",
Expand Down Expand Up @@ -42,13 +41,12 @@
},
{
"@timestamp": "2014-07-08T17:35:27.000Z",
"aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
"aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
"aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
"aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
"aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
"aws.cloudtrail.additional_eventdata.MFAUsed": "No",
"aws.cloudtrail.additional_eventdata.MobileVersion": "No",
"aws.cloudtrail.error_message": "Failed authentication",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
"aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::111122223333:user/JaneDoe",
"aws.cloudtrail.user_identity.type": "IAMUser",
"cloud.account.id": "111122223333",
Expand Down Expand Up @@ -84,13 +82,12 @@
},
{
"@timestamp": "2014-07-08T17:35:27.000Z",
"aws.cloudtrail.additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}",
"aws.cloudtrail.console_login.additional_eventdata.login_to": "https://console.aws.amazon.com/sns",
"aws.cloudtrail.console_login.additional_eventdata.mfa_used": false,
"aws.cloudtrail.console_login.additional_eventdata.mobile_version": false,
"aws.cloudtrail.additional_eventdata.LoginTo": "https://console.aws.amazon.com/sns",
"aws.cloudtrail.additional_eventdata.MFAUsed": "No",
"aws.cloudtrail.additional_eventdata.MobileVersion": "No",
"aws.cloudtrail.error_message": "Failed authentication",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.response_elements": "{ConsoleLogin=Failure}",
"aws.cloudtrail.response_elements.ConsoleLogin": "Failure",
"aws.cloudtrail.user_identity.access_key_id": "AKIAIOSFODNN7EXAMPLE",
"aws.cloudtrail.user_identity.arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName",
"aws.cloudtrail.user_identity.session_context.mfa_authenticated": "false",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
"aws.cloudtrail.event_type": "AwsApiCall",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.recipient_account_id": "0123456789012",
"aws.cloudtrail.request_parameters": "{userName=Bob}",
"aws.cloudtrail.response_elements": "{accessKey={accessKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Active, createDate=Jan 8, 2020 8:43:06 PM}}",
"aws.cloudtrail.request_parameters.userName": "Bob",
"aws.cloudtrail.response_elements.accessKey.accessKeyId": "EXAMPLE_KEY_ID",
"aws.cloudtrail.response_elements.accessKey.createDate": "Jan 8, 2020 8:43:06 PM",
"aws.cloudtrail.response_elements.accessKey.status": "Active",
"aws.cloudtrail.response_elements.accessKey.userName": "Bob",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
"aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
Expand All @@ -26,9 +29,6 @@
"fileset.name": "cloudtrail",
"input.type": "log",
"log.offset": 0,
"related.user": [
"Bob"
],
"service.type": "aws",
"source.address": "127.0.0.1",
"source.ip": "127.0.0.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@
"aws.cloudtrail.event_type": "AwsApiCall",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.recipient_account_id": "0123456789012",
"aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
"aws.cloudtrail.response_elements": "{group={path=/, groupName=TEST-GROUP, groupId=EXAMPLE_ID, arn=arn:aws:iam::0123456789012:group/TEST-GROUP, createDate=Jan 9, 2020 1:48:44 AM}}",
"aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
"aws.cloudtrail.response_elements.group.arn": "arn:aws:iam::0123456789012:group/TEST-GROUP",
"aws.cloudtrail.response_elements.group.createDate": "Jan 9, 2020 1:48:44 AM",
"aws.cloudtrail.response_elements.group.groupId": "EXAMPLE_ID",
"aws.cloudtrail.response_elements.group.groupName": "TEST-GROUP",
"aws.cloudtrail.response_elements.group.path": "/",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
"aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
Expand Down Expand Up @@ -45,7 +49,7 @@
"aws.cloudtrail.event_type": "AwsApiCall",
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.recipient_account_id": "0123456789012",
"aws.cloudtrail.request_parameters": "{groupName=TEST-GROUP}",
"aws.cloudtrail.request_parameters.groupName": "TEST-GROUP",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
"aws.cloudtrail.user_identity.type": "IAMUser",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
{
"@timestamp": "2014-03-06T17:10:34.000Z",
"aws.cloudtrail.event_version": "1.0",
"aws.cloudtrail.request_parameters": "{keyName=mykeypair}",
"aws.cloudtrail.response_elements": "{keyMaterial=<sensitiveDataRemoved>, keyFingerprint=30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21, keyName=mykeypair}",
"aws.cloudtrail.request_parameters.keyName": "mykeypair",
"aws.cloudtrail.response_elements.keyFingerprint": "30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21",
"aws.cloudtrail.response_elements.keyMaterial": "<sensitiveDataRemoved>",
"aws.cloudtrail.response_elements.keyName": "mykeypair",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY_ID",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::123456789012:user/Alice",
"aws.cloudtrail.user_identity.session_context.creation_date": "2014-03-06T15:15:06.000Z",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,20 @@
"aws.cloudtrail.event_version": "1.05",
"aws.cloudtrail.read_only": false,
"aws.cloudtrail.recipient_account_id": "0123456789012",
"aws.cloudtrail.request_parameters": "{isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, enableLogFileValidation=true, kmsKeyId=, isOrganizationTrail=false, includeGlobalServiceEvents=true}",
"aws.cloudtrail.response_elements": "{logFileValidationEnabled=true, isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}",
"aws.cloudtrail.request_parameters.enableLogFileValidation": true,
"aws.cloudtrail.request_parameters.includeGlobalServiceEvents": true,
"aws.cloudtrail.request_parameters.isMultiRegionTrail": true,
"aws.cloudtrail.request_parameters.isOrganizationTrail": false,
"aws.cloudtrail.request_parameters.kmsKeyId": "",
"aws.cloudtrail.request_parameters.name": "TEST-trail",
"aws.cloudtrail.request_parameters.s3BucketName": "TEST-cloudtrail-bucket",
"aws.cloudtrail.response_elements.includeGlobalServiceEvents": true,
"aws.cloudtrail.response_elements.isMultiRegionTrail": true,
"aws.cloudtrail.response_elements.isOrganizationTrail": false,
"aws.cloudtrail.response_elements.logFileValidationEnabled": true,
"aws.cloudtrail.response_elements.name": "TEST-trail",
"aws.cloudtrail.response_elements.s3BucketName": "TEST-cloudtrail-bucket",
"aws.cloudtrail.response_elements.trailARN": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail",
"aws.cloudtrail.user_identity.access_key_id": "EXAMPLE_KEY",
"aws.cloudtrail.user_identity.arn": "arn:aws:iam::0123456789012:user/Alice",
"aws.cloudtrail.user_identity.invoked_by": "signin.amazonaws.com",
Expand Down
Loading

0 comments on commit f004955

Please sign in to comment.