Skip to content

Commit

Permalink
AWS cloudtrail version 0.2.6 (#247)
Browse files Browse the repository at this point in the history 
- add multi_fields to
  aws.cloudtrail.request_parameters
  aws.cloudtrail.response_elements
  aws.cloudtrail.additional_eventdata
  aws.cloudtrail.service_event_details
- add new flattened versions of those fields at:
  aws.cloudtrail.flattened.request_parameters
  aws.cloudtrail.flattened.response_elements
  aws.cloudtrail.flattened.additional_eventdata
  aws.cloudtrail.flattened.service_event_details
- add event.ingested
- move ecs fields to ecs.yml

Closes #167
  • Loading branch information
@leehinman
leehinman authored Aug 20, 2020
1 parent 77a6d37 commit 1901e96
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 114 deletions.
71 changes: 47 additions & 24 deletions packages/aws/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
---
description: Pipeline for AWS CloudTrail Logs
processors:
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
- rename:
field: "message"
target_field: "event.original"
Expand Down Expand Up @@ -84,6 +87,7 @@ processors:
field: "event.action"
value: "{{json.eventName}}"
ignore_failure: true
ignore_empty_value: true
- rename:
field: "json.awsRegion"
target_field: "cloud.region"
Expand Down Expand Up @@ -134,25 +138,37 @@ processors:
field: "json.errorMessage"
target_field: "aws.cloudtrail.error_message"
ignore_failure: true
- rename:
field: json.requestParameters
target_field: "aws.cloudtrail.flattened.request_parameters"
if: ctx.json.requestParameters != null
- script:
lang: painless
source: |
if (ctx.json.requestParameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString();
if (ctx.aws.cloudtrail.flattened.request_parameters != null) {
ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString();
}
ignore_failure: true
- rename:
field: json.responseElements
target_field: "aws.cloudtrail.flattened.response_elements"
if: ctx.json.responseElements != null
- script:
lang: painless
source: |
if (ctx.json.responseElements != null) {
ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString();
if (ctx.aws.cloudtrail.flattened.response_elements != null) {
ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString();
}
ignore_failure: true
- rename:
field: json.additionalEventData
target_field: "aws.cloudtrail.flattened.additional_eventdata"
if: ctx?.json?.additionalEventData != null
- script:
lang: painless
source: |
if (ctx.json.additionalEventData != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString();
if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) {
ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString();
}
ignore_failure: true
- rename:
Expand Down Expand Up @@ -195,11 +211,15 @@ processors:
field: "json.recipientAccountId"
target_field: "aws.cloudtrail.recipient_account_id"
ignore_failure: true
- rename:
field: json.serviceEventDetails
target_field: "aws.cloudtrail.flattened.service_event_details"
if: ctx.json.serviceEventDetails != null
- script:
lang: painless
source: |
if (ctx.json.serviceEventDetails != null) {
ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString();
if (ctx.aws.cloudtrail.flattened.service_event_details != null) {
ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString();
}
ignore_failure: true
- rename:
Expand All @@ -225,14 +245,11 @@ processors:
}
ctx.related.user.add(userName);
}
if (ctx.json?.requestParameters.userName != null) {
addRelatedUser(ctx, ctx.json.requestParameters.userName);
if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName);
}
if (ctx.json?.requestParameters.newUserName != null) {
addRelatedUser(ctx, ctx.json.requestParameters.newUserName);
if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) {
addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName);
}
- script:
Expand All @@ -243,18 +260,18 @@ processors:
return;
}
Map aed_map = new HashMap();
if (ctx.json?.additionalEventData?.MobileVersion != null) {
if (ctx.json.additionalEventData.MobileVersion == 'No') {
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') {
aed_map.put("mobile_version", false);
} else {
aed_map.put("mobile_version", true);
}
}
if (ctx.json?.additionalEventData?.LoginTo != null) {
aed_map.put("login_to", ctx.json.additionalEventData.LoginTo);
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) {
aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo);
}
if (ctx.json?.additionalEventData?.MFAUsed != null) {
if (ctx.json.additionalEventData.MFAUsed == 'No') {
if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) {
if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') {
aed_map.put("mfa_used", false);
} else {
aed_map.put("mfa_used", true);
Expand Down Expand Up @@ -451,6 +468,12 @@ processors:
type:
- user
- info
ListGroupsForUser:
category:
- iam
type:
- user
- info
ListGroupPolicies:
category:
- iam
Expand Down Expand Up @@ -591,10 +614,10 @@ processors:
return;
}
if (ctx.event.action == 'ConsoleLogin' && ctx.json?.responseElements.ConsoleLogin != null) {
ctx.event.outcome = Processors.lowercase(ctx.json.responseElements.ConsoleLogin);
if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) {
ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin);
}
def hm = new HashMap(params.get(ctx.event.action));
hm.forEach((k, v) -> ctx.event[k] = v);
Expand Down
91 changes: 91 additions & 0 deletions packages/aws/dataset/cloudtrail/fields/ecs.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
---
- name: event.action
type: keyword
description: The action captured by the event.
- name: event.original
type: keyword
description: Raw text message of entire event. Used to demonstrate log integrity.
- name: user.name
type: keyword
description: Short name or login of the user.
- name: user.id
type: keyword
description: Unique identifier of the user.
- name: cloud.account.id
type: keyword
description: The cloud account or organization id used to identify different entities
in a multi-tenant environment.
- name: event.provider
type: keyword
description: Source of the event.
- name: cloud.region
type: keyword
description: Region in which this host is running.
- name: source.address
type: keyword
description: Some event source addresses are defined ambiguously. The event will
sometimes list an IP, a domain or a unix socket. You should always store the raw
address in the .address field.
- name: source.ip
type: ip
description: IP address of the source (IPv4 or IPv6).
- name: user_agent.device.name
type: keyword
description: Name of the device.
- name: user_agent.name
type: keyword
description: Name of the user agent.
- name: user_agent.original
type: keyword
description: Unparsed user_agent string.
- name: related.user
type: keyword
description: All the user names seen on your event.
- name: event.kind
type: keyword
description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal)
- name: event.type
type: keyword
description: Event severity (e.g. info, error)
- name: source.as.number
type: long
description: >-
Unique number allocated to the autonomous system. The autonomous
system number (ASN) uniquely identifies each network on the
Internet.
- name: source.as.organization.name
type: keyword
ignore_above: 1024
multi_fields:
- name: text
type: text
norms: false
default_field: false
description: Organization name.
- name: source.geo.city_name
type: keyword
ignore_above: 1024
description: City name.
- name: source.geo.continent_name
type: keyword
ignore_above: 1024
description: Name of the continent.
- name: source.geo.country_iso_code
type: keyword
ignore_above: 1024
description: Country ISO code.
- name: source.geo.country_name
type: keyword
ignore_above: 1024
description: Country name.
- name: source.geo.location
type: geo_point
description: Longitude and latitude.
- name: source.geo.region_iso_code
type: keyword
ignore_above: 1024
description: Region ISO code.
- name: source.geo.region_name
type: keyword
ignore_above: 1024
description: Region name.
130 changes: 41 additions & 89 deletions packages/aws/dataset/cloudtrail/fields/fields.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,26 @@
- name: request_parameters
type: keyword
description: The parameters, if any, that were sent with the request.
multi_fields:
- name: text
type: text
default_field: false
- name: response_elements
type: keyword
description: The response element for actions that make changes (create, update,
or delete actions).
multi_fields:
- name: text
type: text
default_field: false
- name: additional_eventdata
type: keyword
description: Additional data about the event that was not part of the request
or response.
multi_fields:
- name: text
type: text
default_field: false
- name: request_id
type: keyword
description: The value that identifies the request. The service being called generates
Expand Down Expand Up @@ -109,6 +121,10 @@
type: keyword
description: Identifies the service event, including what triggered the event
and the result.
multi_fields:
- name: text
type: text
default_field: false
- name: shared_event_id
type: keyword
description: GUID generated by CloudTrail to uniquely identify CloudTrail events
Expand All @@ -133,92 +149,28 @@
type: boolean
description: Identifies whether multi factor authentication was used during
ConsoleLogin
- name: event.action
type: keyword
description: The action captured by the event.
- name: event.original
type: keyword
description: Raw text message of entire event. Used to demonstrate log integrity.
- name: user.name
type: keyword
description: Short name or login of the user.
- name: user.id
type: keyword
description: Unique identifier of the user.
- name: cloud.account.id
type: keyword
description: The cloud account or organization id used to identify different entities
in a multi-tenant environment.
- name: event.provider
type: keyword
description: Source of the event.
- name: cloud.region
type: keyword
description: Region in which this host is running.
- name: source.address
type: keyword
description: Some event source addresses are defined ambiguously. The event will
sometimes list an IP, a domain or a unix socket. You should always store the raw
address in the .address field.
- name: source.ip
type: ip
description: IP address of the source (IPv4 or IPv6).
- name: user_agent.device.name
type: keyword
description: Name of the device.
- name: user_agent.name
type: keyword
description: Name of the user agent.
- name: user_agent.original
type: keyword
description: Unparsed user_agent string.
- name: related.user
type: keyword
description: All the user names seen on your event.
- name: event.kind
type: keyword
description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal)
- name: event.type
type: keyword
description: Event severity (e.g. info, error)
- name: source.as.number
type: long
description: >-
Unique number allocated to the autonomous system. The autonomous system number
(ASN) uniquely identifies each network on the Internet.
- name: source.as.organization.name
type: keyword
ignore_above: 1024
multi_fields:
- name: text
type: text
norms: false
default_field: false
description: Organization name.
- name: source.geo.city_name
type: keyword
ignore_above: 1024
description: City name.
- name: source.geo.continent_name
type: keyword
ignore_above: 1024
description: Name of the continent.
- name: source.geo.country_iso_code
type: keyword
ignore_above: 1024
description: Country ISO code.
- name: source.geo.country_name
type: keyword
ignore_above: 1024
description: Country name.
- name: source.geo.location
type: geo_point
description: Longitude and latitude.
- name: source.geo.region_iso_code
type: keyword
ignore_above: 1024
description: Region ISO code.
- name: source.geo.region_name
type: keyword
ignore_above: 1024
description: Region name.
- name: flattened
type: group
description: >-
ES flattened datatype for objects where the subfields aren't
known in advance.
fields:
- name: additional_eventdata
type: flattened
description: >
Additional data about the event that was not part of the
request or response.
- name: request_parameters
type: flattened
description: >-
The parameters, if any, that were sent with the request.
- name: response_elements
type: flattened
description: >-
The response element for actions that make changes (create,
update, or delete actions).
- name: service_event_details
type: flattened
description: >-
Identifies the service event, including what triggered the
event and the result.
Loading

0 comments on commit 1901e96

Please sign in to comment.