Skip to content

Commit

Permalink
Sync azure package with beats (elastic#722)
Browse files Browse the repository at this point in the history
* Sync azure package with beats

* Add changelog
  • Loading branch information
marc-gr authored Feb 18, 2021
1 parent 75ceb9e commit 2807fa9
Show file tree
Hide file tree
Showing 23 changed files with 179 additions and 87 deletions.
5 changes: 5 additions & 0 deletions packages/azure/changelog.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "0.2.0"
changes:
- description: Add changes to use ECS 1.8 fields.
type: enhancement # can be one of: enhancement, bugfix, breaking-change
link: https://github.com/elastic/integrations/pull/722
- version: "0.0.1"
changes:
- description: initial release
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
{
"expected": [
{
"log": {
"level": "Information"
},
"source": {
"geo": {
"continent_name": "Europe",
"country_name": "United Kingdom",
"location": {
"lon": -0.1224,
"lat": 51.4964
},
"country_iso_code": "GB"
},
"ip": "51.251.141.41"
},
"geo": {
"continent_name": "Europe",
"country_name": "United Kingdom",
Expand All @@ -14,28 +29,22 @@
"provider": "azure"
},
"@timestamp": "2019-10-24T00:13:46.355Z",
"related": {
"ip": [
"51.251.141.41"
]
},
"ecs": {
"version": "1.5.0"
},
"log": {
"level": "Information"
},
"source": {
"geo": {
"continent_name": "Europe",
"country_name": "United Kingdom",
"location": {
"lon": -0.1224,
"lat": 51.4964
},
"country_iso_code": "GB"
},
"client": {
"ip": "51.251.141.41"
},
"event": {
"duration": 0,
"action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION",
"ingested": "2020-12-04T13:40:57.414565400Z",
"ingested": "2021-02-17T15:44:41.246811100Z",
"original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}",
"type": [
"change"
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@ processors:
ignore_failure: true
formats:
- ISO8601
- rename:
field: message
target_field: event.original
- remove:
field:
- message
- azure.activitylogs.time
field: azure.activitylogs.time
ignore_missing: true
- rename:
field: azure.activitylogs.resourceId
Expand All @@ -35,6 +36,15 @@ processors:
field: azure.activitylogs.callerIpAddress
target_field: source.ip
ignore_missing: true
- set:
field: client.ip
value: '{{source.ip}}'
ignore_empty_value: true
- append:
field: related.ip
value: '{{source.ip}}'
allow_duplicates: false
if: 'ctx.source?.ip != null'
- rename:
field: azure.activitylogs.level
target_field: log.level
Expand Down Expand Up @@ -224,6 +234,26 @@ processors:
patterns:
- '%{USERNAME:user.name}@%{HOSTNAME:user.domain}'
ignore_missing: true
ignore_failure: true

# set user.email to the original name if the above grok succeeded.
- set:
field: user.email
value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}'
ignore_empty_value: true
if: 'ctx.user?.name != null'

# set user.name to the original name if the above grok failed (name format is not an email).
- set:
field: user.name
value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}'
ignore_empty_value: true
if: 'ctx.user?.name == null'
- append:
field: related.user
value: '{{user.name}}'
allow_duplicates: false
if: 'ctx.user?.name != null'
- convert:
field: azure.activitylogs.identity.claims_initiated_by_user.fullname
target_field: user.full_name
Expand Down
3 changes: 3 additions & 0 deletions packages/azure/data_stream/activitylogs/fields/ecs.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
- description: IP address of the client.
name: client.ip
type: ip
- description: Destination network address.
ignore_above: 1024
name: destination.address
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"event": {
"duration": 0,
"action": "Update device",
"ingested": "2020-12-04T13:40:57.990898700Z",
"ingested": "2021-02-17T15:44:41.925301300Z",
"original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}",
"kind": "event",
"outcome": "success"
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,11 @@ processors:
field: azure.auditlogs.level
target_field: log.level
ignore_missing: true
- rename:
field: message
target_field: event.original
- remove:
field:
- message
- azure.auditlogs.time
field: azure.auditlogs.time
ignore_missing: true
- convert:
field: azure.auditlogs.operationName
Expand Down
Original file line number Diff line number Diff line change
@@ -1,28 +1,12 @@
{
"expected": [
{
"cloud": {
"provider": "azure"
},
"ecs": {
"version": "1.5.0"
},
"message": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}",
"event": {
"ingested": "2020-12-04T13:40:58.273944800Z",
"kind": "event"
"ingested": "2021-02-17T15:44:42.249938900Z"
},
"error": {
"message": "invalid json log"
},
"azure": {
"resource": {
"id": "/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY"
},
"platformlogs": {
"category": "ApplicationGatewayAccessLog",
"event_category": "Administrative"
}
"message": "Unexpected character ('M' (code 77)): was expecting comma to separate Object entries\\n at [Source: (org.elasticsearch.common.io.stream.InputStreamStreamInput); line: 1, column: 509]"
}
}
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
},
"event": {
"action": "Retreive ConsumerGroup",
"ingested": "2020-12-04T13:40:58.303070600Z",
"ingested": "2021-02-17T15:44:42.262500900Z",
"original": "{\"ActivityId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8\",\"Caller\":\"Portal\",\"Environment\":\"PROD\",\"EventName\":\"Retreive ConsumerGroup\",\"EventProperties\":\"{\\\"SubscriptionId\\\":\\\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\\\",\\\"Namespace\\\":\\\"obstesteventhubs\\\",\\\"Via\\\":\\\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\\u0026$skip=0\\u0026$top=100\\\",\\\"TrackingId\\\":\\\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\\\"}\",\"EventTimeString\":\"11/3/2020 9:06:42 AM +00:00\",\"Region\":\"West Europe\",\"ScaleUnit\":\"PROD-AM3-AZ501\",\"Status\":\"Succeeded\",\"category\":\"OperationalLogs\",\"resourceId\":\"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS\"}",
"kind": "event",
"outcome": "succeeded"
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
},
"event": {
"action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read",
"ingested": "2020-12-04T13:40:58.343665Z",
"ingested": "2021-02-17T15:44:42.305137300Z",
"original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}",
"kind": "event"
},
"message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,9 @@ tags:
{{/each}}
{{#contains tags "forwarded"}}
publisher_pipeline.disable_host: true
{{/contains}}
{{/contains}}
processors:
- add_fields:
target: ''
fields:
ecs.version: 1.8.0
Original file line number Diff line number Diff line change
Expand Up @@ -16,23 +16,6 @@ processors:
- json:
field: message
target_field: azure.platformlogs
on_failure:
- grok:
field: message
patterns:
- "resourceId\": \"%{DATA:azure.platformlogs.resourceId}\""
ignore_failure: true
ignore_missing: true
- grok:
field: message
patterns:
- "category\": \"%{DATA:azure.platformlogs.category}\""
ignore_failure: true
ignore_missing: true
- set:
field: error.message
value: 'invalid json log'
ignore_failure: true
- date:
field: azure.platformlogs.time
target_field: '@timestamp'
Expand All @@ -46,14 +29,11 @@ processors:
formats:
- ISO8601
- "M/d/yyyy h:mm:ss a XXX"
- rename:
field: message
target_field: event.original
- remove:
if: "ctx.error?.message != 'invalid json log'"
field:
- message
ignore_missing: true
- remove:
field:
- azure.platformlogs.time
field: azure.platformlogs.time
ignore_missing: true
- rename:
field: azure.platformlogs.resourceId
Expand Down Expand Up @@ -84,6 +64,15 @@ processors:
field: azure.platformlogs.callerIpAddress
target_field: source.ip
ignore_missing: true
- set:
field: client.ip
value: '{{source.ip}}'
ignore_empty_value: true
- append:
field: related.ip
value: '{{source.ip}}'
allow_duplicates: false
if: 'ctx.source?.ip != null'
- rename:
field: azure.platformlogs.level
target_field: log.level
Expand Down Expand Up @@ -124,19 +113,16 @@ processors:
field: azure.platformlogs.result_type
target_field: event.outcome
type: string
ignore_missing: true
if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')"
- convert:
field: azure.platformlogs.properties.result
target_field: event.outcome
type: string
if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)"
ignore_missing: true
- convert:
field: azure.platformlogs.Status
target_field: event.outcome
type: string
ignore_missing: true
if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)"
- rename:
field: azure.platformlogs.operationName
Expand Down Expand Up @@ -212,11 +198,9 @@ processors:
- set:
field: event.kind
value: event
ignore_failure: true
- pipeline:
name: '{{ IngestPipeline "azure-shared-pipeline" }}'
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
ignore_failure: true
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
Loading

0 comments on commit 2807fa9

Please sign in to comment.