diff --git a/packages/aws/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml index 66e8110d968..42395228853 100644 --- a/packages/aws/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/dataset/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -84,6 +87,7 @@ processors: field: "event.action" value: "{{json.eventName}}" ignore_failure: true + ignore_empty_value: true - rename: field: "json.awsRegion" target_field: "cloud.region" @@ -134,25 +138,37 @@ processors: field: "json.errorMessage" target_field: "aws.cloudtrail.error_message" ignore_failure: true + - rename: + field: json.requestParameters + target_field: "aws.cloudtrail.flattened.request_parameters" + if: ctx.json.requestParameters != null - script: lang: painless source: | - if (ctx.json.requestParameters != null) { - ctx.aws.cloudtrail.request_parameters = ctx.json.requestParameters.toString(); + if (ctx.aws.cloudtrail.flattened.request_parameters != null) { + ctx.aws.cloudtrail.request_parameters = ctx.aws.cloudtrail.flattened.request_parameters.toString(); } ignore_failure: true + - rename: + field: json.responseElements + target_field: "aws.cloudtrail.flattened.response_elements" + if: ctx.json.responseElements != null - script: lang: painless source: | - if (ctx.json.responseElements != null) { - ctx.aws.cloudtrail.response_elements = ctx.json.responseElements.toString(); + if (ctx.aws.cloudtrail.flattened.response_elements != null) { + ctx.aws.cloudtrail.response_elements = ctx.aws.cloudtrail.flattened.response_elements.toString(); } ignore_failure: true + - rename: + field: json.additionalEventData + target_field: "aws.cloudtrail.flattened.additional_eventdata" + if: ctx?.json?.additionalEventData != null - script: lang: painless source: | - if (ctx.json.additionalEventData != null) { - ctx.aws.cloudtrail.additional_eventdata = ctx.json.additionalEventData.toString(); + if (ctx.aws.cloudtrail.flattened.additional_eventdata != null) { + ctx.aws.cloudtrail.additional_eventdata = ctx.aws.cloudtrail.flattened.additional_eventdata.toString(); } ignore_failure: true - rename: @@ -195,11 +211,15 @@ processors: field: "json.recipientAccountId" target_field: "aws.cloudtrail.recipient_account_id" ignore_failure: true + - rename: + field: json.serviceEventDetails + target_field: "aws.cloudtrail.flattened.service_event_details" + if: ctx.json.serviceEventDetails != null - script: lang: painless source: | - if (ctx.json.serviceEventDetails != null) { - ctx.aws.cloudtrail.service_event_details = ctx.json.serviceEventDetails.toString(); + if (ctx.aws.cloudtrail.flattened.service_event_details != null) { + ctx.aws.cloudtrail.service_event_details = ctx.aws.cloudtrail.flattened.service_event_details.toString(); } ignore_failure: true - rename: @@ -225,14 +245,11 @@ processors: } ctx.related.user.add(userName); } - - - - if (ctx.json?.requestParameters.userName != null) { - addRelatedUser(ctx, ctx.json.requestParameters.userName); + if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.userName != null) { + addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.userName); } - if (ctx.json?.requestParameters.newUserName != null) { - addRelatedUser(ctx, ctx.json.requestParameters.newUserName); + if (ctx?.aws?.cloudtrail?.flattened?.request_parameters?.newUserName != null) { + addRelatedUser(ctx, ctx.aws.cloudtrail.flattened.request_parameters.newUserName); } - script: @@ -243,18 +260,18 @@ processors: return; } Map aed_map = new HashMap(); - if (ctx.json?.additionalEventData?.MobileVersion != null) { - if (ctx.json.additionalEventData.MobileVersion == 'No') { + if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MobileVersion != null) { + if (ctx.aws.cloudtrail.flattened.additional_eventdata.MobileVersion == 'No') { aed_map.put("mobile_version", false); } else { aed_map.put("mobile_version", true); } } - if (ctx.json?.additionalEventData?.LoginTo != null) { - aed_map.put("login_to", ctx.json.additionalEventData.LoginTo); + if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.LoginTo != null) { + aed_map.put("login_to", ctx.aws.cloudtrail.flattened.additional_eventdata.LoginTo); } - if (ctx.json?.additionalEventData?.MFAUsed != null) { - if (ctx.json.additionalEventData.MFAUsed == 'No') { + if (ctx?.aws?.cloudtrail?.flattened?.additional_eventdata?.MFAUsed != null) { + if (ctx.aws.cloudtrail.flattened.additional_eventdata.MFAUsed == 'No') { aed_map.put("mfa_used", false); } else { aed_map.put("mfa_used", true); @@ -451,6 +468,12 @@ processors: type: - user - info + ListGroupsForUser: + category: + - iam + type: + - user + - info ListGroupPolicies: category: - iam @@ -591,10 +614,10 @@ processors: return; } - if (ctx.event.action == 'ConsoleLogin' && ctx.json?.responseElements.ConsoleLogin != null) { - ctx.event.outcome = Processors.lowercase(ctx.json.responseElements.ConsoleLogin); + if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { + ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/packages/aws/dataset/cloudtrail/fields/ecs.yml b/packages/aws/dataset/cloudtrail/fields/ecs.yml new file mode 100644 index 00000000000..e182a694b18 --- /dev/null +++ b/packages/aws/dataset/cloudtrail/fields/ecs.yml @@ -0,0 +1,91 @@ +--- +- name: event.action + type: keyword + description: The action captured by the event. +- name: event.original + type: keyword + description: Raw text message of entire event. Used to demonstrate log integrity. +- name: user.name + type: keyword + description: Short name or login of the user. +- name: user.id + type: keyword + description: Unique identifier of the user. +- name: cloud.account.id + type: keyword + description: The cloud account or organization id used to identify different entities + in a multi-tenant environment. +- name: event.provider + type: keyword + description: Source of the event. +- name: cloud.region + type: keyword + description: Region in which this host is running. +- name: source.address + type: keyword + description: Some event source addresses are defined ambiguously. The event will + sometimes list an IP, a domain or a unix socket. You should always store the raw + address in the .address field. +- name: source.ip + type: ip + description: IP address of the source (IPv4 or IPv6). +- name: user_agent.device.name + type: keyword + description: Name of the device. +- name: user_agent.name + type: keyword + description: Name of the user agent. +- name: user_agent.original + type: keyword + description: Unparsed user_agent string. +- name: related.user + type: keyword + description: All the user names seen on your event. +- name: event.kind + type: keyword + description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) +- name: event.type + type: keyword + description: Event severity (e.g. info, error) +- name: source.as.number + type: long + description: >- + Unique number allocated to the autonomous system. The autonomous + system number (ASN) uniquely identifies each network on the + Internet. +- name: source.as.organization.name + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Organization name. +- name: source.geo.city_name + type: keyword + ignore_above: 1024 + description: City name. +- name: source.geo.continent_name + type: keyword + ignore_above: 1024 + description: Name of the continent. +- name: source.geo.country_iso_code + type: keyword + ignore_above: 1024 + description: Country ISO code. +- name: source.geo.country_name + type: keyword + ignore_above: 1024 + description: Country name. +- name: source.geo.location + type: geo_point + description: Longitude and latitude. +- name: source.geo.region_iso_code + type: keyword + ignore_above: 1024 + description: Region ISO code. +- name: source.geo.region_name + type: keyword + ignore_above: 1024 + description: Region name. diff --git a/packages/aws/dataset/cloudtrail/fields/fields.yml b/packages/aws/dataset/cloudtrail/fields/fields.yml index fc6eee231ae..bf37d78d564 100644 --- a/packages/aws/dataset/cloudtrail/fields/fields.yml +++ b/packages/aws/dataset/cloudtrail/fields/fields.yml @@ -64,14 +64,26 @@ - name: request_parameters type: keyword description: The parameters, if any, that were sent with the request. + multi_fields: + - name: text + type: text + default_field: false - name: response_elements type: keyword description: The response element for actions that make changes (create, update, or delete actions). + multi_fields: + - name: text + type: text + default_field: false - name: additional_eventdata type: keyword description: Additional data about the event that was not part of the request or response. + multi_fields: + - name: text + type: text + default_field: false - name: request_id type: keyword description: The value that identifies the request. The service being called generates @@ -109,6 +121,10 @@ type: keyword description: Identifies the service event, including what triggered the event and the result. + multi_fields: + - name: text + type: text + default_field: false - name: shared_event_id type: keyword description: GUID generated by CloudTrail to uniquely identify CloudTrail events @@ -133,92 +149,28 @@ type: boolean description: Identifies whether multi factor authentication was used during ConsoleLogin -- name: event.action - type: keyword - description: The action captured by the event. -- name: event.original - type: keyword - description: Raw text message of entire event. Used to demonstrate log integrity. -- name: user.name - type: keyword - description: Short name or login of the user. -- name: user.id - type: keyword - description: Unique identifier of the user. -- name: cloud.account.id - type: keyword - description: The cloud account or organization id used to identify different entities - in a multi-tenant environment. -- name: event.provider - type: keyword - description: Source of the event. -- name: cloud.region - type: keyword - description: Region in which this host is running. -- name: source.address - type: keyword - description: Some event source addresses are defined ambiguously. The event will - sometimes list an IP, a domain or a unix socket. You should always store the raw - address in the .address field. -- name: source.ip - type: ip - description: IP address of the source (IPv4 or IPv6). -- name: user_agent.device.name - type: keyword - description: Name of the device. -- name: user_agent.name - type: keyword - description: Name of the user agent. -- name: user_agent.original - type: keyword - description: Unparsed user_agent string. -- name: related.user - type: keyword - description: All the user names seen on your event. -- name: event.kind - type: keyword - description: Event kind (e.g. event, alert, metric, state, pipeline_error, signal) -- name: event.type - type: keyword - description: Event severity (e.g. info, error) -- name: source.as.number - type: long - description: >- - Unique number allocated to the autonomous system. The autonomous system number - (ASN) uniquely identifies each network on the Internet. -- name: source.as.organization.name - type: keyword - ignore_above: 1024 - multi_fields: - - name: text - type: text - norms: false - default_field: false - description: Organization name. -- name: source.geo.city_name - type: keyword - ignore_above: 1024 - description: City name. -- name: source.geo.continent_name - type: keyword - ignore_above: 1024 - description: Name of the continent. -- name: source.geo.country_iso_code - type: keyword - ignore_above: 1024 - description: Country ISO code. -- name: source.geo.country_name - type: keyword - ignore_above: 1024 - description: Country name. -- name: source.geo.location - type: geo_point - description: Longitude and latitude. -- name: source.geo.region_iso_code - type: keyword - ignore_above: 1024 - description: Region ISO code. -- name: source.geo.region_name - type: keyword - ignore_above: 1024 - description: Region name. + - name: flattened + type: group + description: >- + ES flattened datatype for objects where the subfields aren't + known in advance. + fields: + - name: additional_eventdata + type: flattened + description: > + Additional data about the event that was not part of the + request or response. + - name: request_parameters + type: flattened + description: >- + The parameters, if any, that were sent with the request. + - name: response_elements + type: flattened + description: >- + The response element for actions that make changes (create, + update, or delete actions). + - name: service_event_details + type: flattened + description: >- + Identifies the service event, including what triggered the + event and the result. diff --git a/packages/aws/docs/README.md b/packages/aws/docs/README.md index 31dab985d35..15a68fd639e 100644 --- a/packages/aws/docs/README.md +++ b/packages/aws/docs/README.md @@ -27,6 +27,10 @@ events for the account. If user creates a trail, it delivers those events as log | aws.cloudtrail.error_message | If the request returns an error, the description of the error. | keyword | | aws.cloudtrail.event_type | Identifies the type of event that generated the event record. | keyword | | aws.cloudtrail.event_version | The CloudTrail version of the log event format. | keyword | +| aws.cloudtrail.flattened.additional_eventdata | Additional data about the event that was not part of the request or response. | flattened | +| aws.cloudtrail.flattened.request_parameters | The parameters, if any, that were sent with the request. | flattened | +| aws.cloudtrail.flattened.response_elements | The response element for actions that make changes (create, update, or delete actions). | flattened | +| aws.cloudtrail.flattened.service_event_details | Identifies the service event, including what triggered the event and the result. | flattened | | aws.cloudtrail.management_event | A Boolean value that identifies whether the event is a management event. | keyword | | aws.cloudtrail.read_only | Identifies whether this operation is a read-only operation. | keyword | | aws.cloudtrail.recipient_account_id | Represents the account ID that received this event. | keyword | diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index c915042988c..c44d83a2b6d 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.2.5 +version: 0.2.6 license: basic description: AWS Integration type: integration