From f4df685a0c36b9c592a91ddc5aa0a1dcca66b1bb Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Wed, 17 Feb 2021 16:45:05 +0100 Subject: [PATCH 1/2] Sync azure package with beats --- .../test-activitylogs-raw.log-expected.json | 37 ++++++++------ .../agent/stream/azure-eventhub.yml.hbs | 7 ++- .../activitylogs/agent/stream/log.yml.hbs | 7 ++- .../elasticsearch/ingest_pipeline/default.yml | 36 ++++++++++++-- .../data_stream/activitylogs/fields/ecs.yml | 3 ++ .../test-auditlogs-raw.log-expected.json | 3 +- .../agent/stream/azure-eventhub.yml.hbs | 7 ++- .../auditlogs/agent/stream/log.yml.hbs | 7 ++- .../elasticsearch/ingest_pipeline/default.yml | 7 +-- ...platformlogs-invalid-raw.log-expected.json | 20 +------- .../test-platformlogs-raw.log-expected.json | 3 +- ...-platformlogs-remote-raw.log-expected.json | 3 +- .../agent/stream/azure-eventhub.yml.hbs | 7 ++- .../platformlogs/agent/stream/log.yml.hbs | 7 ++- .../elasticsearch/ingest_pipeline/default.yml | 48 +++++++------------ .../test-signinlogs-raw.log-expected.json | 22 ++++++++- .../agent/stream/azure-eventhub.yml.hbs | 7 ++- .../signinlogs/agent/stream/log.yml.hbs | 7 ++- .../elasticsearch/ingest_pipeline/default.yml | 16 +++++-- .../data_stream/signinlogs/fields/ecs.yml | 3 ++ packages/azure/docs/README.md | 2 + packages/azure/manifest.yml | 2 +- 22 files changed, 174 insertions(+), 87 deletions(-) diff --git a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json index 6b1081c1f31..1a4bb1234c6 100644 --- a/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json +++ b/packages/azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json @@ -1,6 +1,21 @@ { "expected": [ { + "log": { + "level": "Information" + }, + "source": { + "geo": { + "continent_name": "Europe", + "country_name": "United Kingdom", + "location": { + "lon": -0.1224, + "lat": 51.4964 + }, + "country_iso_code": "GB" + }, + "ip": "51.251.141.41" + }, "geo": { "continent_name": "Europe", "country_name": "United Kingdom", @@ -14,28 +29,22 @@ "provider": "azure" }, "@timestamp": "2019-10-24T00:13:46.355Z", + "related": { + "ip": [ + "51.251.141.41" + ] + }, "ecs": { "version": "1.5.0" }, - "log": { - "level": "Information" - }, - "source": { - "geo": { - "continent_name": "Europe", - "country_name": "United Kingdom", - "location": { - "lon": -0.1224, - "lat": 51.4964 - }, - "country_iso_code": "GB" - }, + "client": { "ip": "51.251.141.41" }, "event": { "duration": 0, "action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "ingested": "2020-12-04T13:40:57.414565400Z", + "ingested": "2021-02-17T15:44:41.246811100Z", + "original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}", "type": [ "change" ], diff --git a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs index 8009e9c32c0..3330f697fc3 100644 --- a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs @@ -11,4 +11,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs index f60972ae6db..d4d30f1f102 100644 --- a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs @@ -9,4 +9,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml index e7990fbc09a..55778f210e0 100644 --- a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml @@ -22,10 +22,11 @@ processors: ignore_failure: true formats: - ISO8601 + - rename: + field: message + target_field: event.original - remove: - field: - - message - - azure.activitylogs.time + field: azure.activitylogs.time ignore_missing: true - rename: field: azure.activitylogs.resourceId @@ -35,6 +36,15 @@ processors: field: azure.activitylogs.callerIpAddress target_field: source.ip ignore_missing: true + - set: + field: client.ip + value: '{{source.ip}}' + ignore_empty_value: true + - append: + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + if: 'ctx.source?.ip != null' - rename: field: azure.activitylogs.level target_field: log.level @@ -224,6 +234,26 @@ processors: patterns: - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}' ignore_missing: true + ignore_failure: true + + # set user.email to the original name if the above grok succeeded. + - set: + field: user.email + value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}' + ignore_empty_value: true + if: 'ctx.user?.name != null' + + # set user.name to the original name if the above grok failed (name format is not an email). + - set: + field: user.name + value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}' + ignore_empty_value: true + if: 'ctx.user?.name == null' + - append: + field: related.user + value: '{{user.name}}' + allow_duplicates: false + if: 'ctx.user?.name != null' - convert: field: azure.activitylogs.identity.claims_initiated_by_user.fullname target_field: user.full_name diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml index 6ed7adae3a8..11efd619e1e 100644 --- a/packages/azure/data_stream/activitylogs/fields/ecs.yml +++ b/packages/azure/data_stream/activitylogs/fields/ecs.yml @@ -1,3 +1,6 @@ +- description: IP address of the client. + name: client.ip + type: ip - description: Destination network address. ignore_above: 1024 name: destination.address diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json index 204bdecff26..b69f30a49d4 100644 --- a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json +++ b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json @@ -14,7 +14,8 @@ "event": { "duration": 0, "action": "Update device", - "ingested": "2020-12-04T13:40:57.990898700Z", + "ingested": "2021-02-17T15:44:41.925301300Z", + "original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}", "kind": "event", "outcome": "success" }, diff --git a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs index 7a6e08b4415..8d7487d52c9 100644 --- a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs @@ -11,4 +11,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs index f60972ae6db..d4d30f1f102 100644 --- a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs @@ -9,4 +9,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml index eeff22f8718..68873c59fc1 100644 --- a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml @@ -40,10 +40,11 @@ processors: field: azure.auditlogs.level target_field: log.level ignore_missing: true + - rename: + field: message + target_field: event.original - remove: - field: - - message - - azure.auditlogs.time + field: azure.auditlogs.time ignore_missing: true - convert: field: azure.auditlogs.operationName diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json index d85d393c378..0af0d546a2e 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json @@ -1,28 +1,12 @@ { "expected": [ { - "cloud": { - "provider": "azure" - }, - "ecs": { - "version": "1.5.0" - }, "message": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}", "event": { - "ingested": "2020-12-04T13:40:58.273944800Z", - "kind": "event" + "ingested": "2021-02-17T15:44:42.249938900Z" }, "error": { - "message": "invalid json log" - }, - "azure": { - "resource": { - "id": "/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY" - }, - "platformlogs": { - "category": "ApplicationGatewayAccessLog", - "event_category": "Administrative" - } + "message": "Unexpected character ('M' (code 77)): was expecting comma to separate Object entries\\n at [Source: (org.elasticsearch.common.io.stream.InputStreamStreamInput); line: 1, column: 509]" } } ] diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json index d2ce6e1cb26..346d7a792e6 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json @@ -11,7 +11,8 @@ }, "event": { "action": "Retreive ConsumerGroup", - "ingested": "2020-12-04T13:40:58.303070600Z", + "ingested": "2021-02-17T15:44:42.262500900Z", + "original": "{\"ActivityId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8\",\"Caller\":\"Portal\",\"Environment\":\"PROD\",\"EventName\":\"Retreive ConsumerGroup\",\"EventProperties\":\"{\\\"SubscriptionId\\\":\\\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\\\",\\\"Namespace\\\":\\\"obstesteventhubs\\\",\\\"Via\\\":\\\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\\u0026$skip=0\\u0026$top=100\\\",\\\"TrackingId\\\":\\\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\\\"}\",\"EventTimeString\":\"11/3/2020 9:06:42 AM +00:00\",\"Region\":\"West Europe\",\"ScaleUnit\":\"PROD-AM3-AZ501\",\"Status\":\"Succeeded\",\"category\":\"OperationalLogs\",\"resourceId\":\"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS\"}", "kind": "event", "outcome": "succeeded" }, diff --git a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json index fc0fd035d9a..1230774d531 100644 --- a/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json +++ b/packages/azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json @@ -10,7 +10,8 @@ }, "event": { "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested": "2020-12-04T13:40:58.343665Z", + "ingested": "2021-02-17T15:44:42.305137300Z", + "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}", "kind": "event" }, "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", diff --git a/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs index 8f082a4fb55..ac0af8e537f 100644 --- a/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs @@ -11,4 +11,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs index f60972ae6db..d4d30f1f102 100644 --- a/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs @@ -9,4 +9,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml index 529ad5cc77f..2cb2c9aba33 100644 --- a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml @@ -16,23 +16,6 @@ processors: - json: field: message target_field: azure.platformlogs - on_failure: - - grok: - field: message - patterns: - - "resourceId\": \"%{DATA:azure.platformlogs.resourceId}\"" - ignore_failure: true - ignore_missing: true - - grok: - field: message - patterns: - - "category\": \"%{DATA:azure.platformlogs.category}\"" - ignore_failure: true - ignore_missing: true - - set: - field: error.message - value: 'invalid json log' - ignore_failure: true - date: field: azure.platformlogs.time target_field: '@timestamp' @@ -46,14 +29,11 @@ processors: formats: - ISO8601 - "M/d/yyyy h:mm:ss a XXX" + - rename: + field: message + target_field: event.original - remove: - if: "ctx.error?.message != 'invalid json log'" - field: - - message - ignore_missing: true - - remove: - field: - - azure.platformlogs.time + field: azure.platformlogs.time ignore_missing: true - rename: field: azure.platformlogs.resourceId @@ -84,6 +64,15 @@ processors: field: azure.platformlogs.callerIpAddress target_field: source.ip ignore_missing: true + - set: + field: client.ip + value: '{{source.ip}}' + ignore_empty_value: true + - append: + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + if: 'ctx.source?.ip != null' - rename: field: azure.platformlogs.level target_field: log.level @@ -124,19 +113,16 @@ processors: field: azure.platformlogs.result_type target_field: event.outcome type: string - ignore_missing: true if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')" - convert: field: azure.platformlogs.properties.result target_field: event.outcome type: string if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)" - ignore_missing: true - convert: field: azure.platformlogs.Status target_field: event.outcome type: string - ignore_missing: true if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)" - rename: field: azure.platformlogs.operationName @@ -212,11 +198,9 @@ processors: - set: field: event.kind value: event - ignore_failure: true - pipeline: name: '{{ IngestPipeline "azure-shared-pipeline" }}' on_failure: - - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' - ignore_failure: true \ No newline at end of file +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json index 330cdd5cfdb..8a2780bdab1 100644 --- a/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json +++ b/packages/azure/data_stream/signinlogs/_dev/test/pipeline/test-signinlogs-raw.log-expected.json @@ -39,12 +39,21 @@ "provider": "azure" }, "@timestamp": "2019-10-18T09:45:48.072Z", + "related": { + "ip": [ + "81.171.241.231" + ] + }, "ecs": { "version": "1.5.0" }, + "client": { + "ip": "81.171.241.231" + }, "event": { "duration": 0, - "ingested": "2020-12-04T13:40:58.570100900Z", + "ingested": "2021-02-17T15:44:42.539291Z", + "original": "{\"Level\":\"4\",\"callerIpAddress\":\"81.171.241.231\",\"category\":\"SignInLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"test@elastic.co\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", "category": [ @@ -146,12 +155,21 @@ "provider": "azure" }, "@timestamp": "2019-10-18T09:45:48.072Z", + "related": { + "ip": [ + "8.8.8.8" + ] + }, "ecs": { "version": "1.5.0" }, + "client": { + "ip": "8.8.8.8" + }, "event": { "duration": 0, - "ingested": "2020-12-04T13:40:58.570119600Z", + "ingested": "2021-02-17T15:44:42.539309700Z", + "original": "{\"Level\":\"4\",\"callerIpAddress\":\"8.8.8.8\",\"category\":\"SignInLogs\",\"correlationId\":\"a8d4eb85-90c5-740d-9af6-7a15036cd135\",\"durationMs\":0,\"identity\":\"Test LTest\",\"location\":\"FR\",\"operationName\":\"Sign-in activity\",\"operationVersion\":\"1.0\",\"properties\":{\"appDisplayName\":\"Office 365\",\"appId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"clientAppUsed\":\"Browser\",\"conditionalAccessStatus\":\"notApplied\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"createdDateTime\":\"2019-10-18T04:45:48.0729893-05:00\",\"deviceDetail\":{\"browser\":\"Chrome 77.0.3865\",\"deviceId\":\"\",\"operatingSystem\":\"MacOs\"},\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ipAddress\":\"81.171.241.231\",\"isInteractive\":false,\"location\":{\"city\":\"Champs-Sur-Marne\",\"countryOrRegion\":\"FR\",\"geoCoordinates\":{\"latitude\":48.12341234,\"longitude\":2.12341234},\"state\":\"Seine-Et-Marne\"},\"originalRequestId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"processingTimeInMilliseconds\":239,\"riskDetail\":\"none\",\"riskLevelAggregated\":\"none\",\"riskLevelDuringSignIn\":\"none\",\"riskState\":\"none\",\"servicePrincipalId\":\"\",\"status\":{\"errorCode\":50140,\"failureReason\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\"},\"tokenIssuerName\":\"\",\"tokenIssuerType\":\"AzureAD\",\"userDisplayName\":\"Test LTest\",\"userId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"userPrincipalName\":\"c3813493-bf92-5123-2717-8a8b2979c38b\"},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultDescription\":\"This error occurred due to 'Keep me signed in' interrupt when the user was signing-in.\",\"resultSignature\":\"None\",\"resultType\":\"50140\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T09:45:48.0729893Z\"}", "kind": "event", "action": "Sign-in activity", "category": [ diff --git a/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs index b3b1df33cc6..087f3ef1306 100644 --- a/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs +++ b/packages/azure/data_stream/signinlogs/agent/stream/azure-eventhub.yml.hbs @@ -11,4 +11,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs index f60972ae6db..d4d30f1f102 100644 --- a/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs +++ b/packages/azure/data_stream/signinlogs/agent/stream/log.yml.hbs @@ -9,4 +9,9 @@ tags: {{/each}} {{#contains tags "forwarded"}} publisher_pipeline.disable_host: true -{{/contains}} \ No newline at end of file +{{/contains}} +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.8.0 \ No newline at end of file diff --git a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml index 2a7923b4e84..2a2ca3942c2 100644 --- a/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure/data_stream/signinlogs/elasticsearch/ingest_pipeline/default.yml @@ -19,10 +19,11 @@ processors: ignore_failure: false formats: - ISO8601 + - rename: + field: message + target_field: event.original - remove: - field: - - message - - azure.signinlogs.time + field: azure.signinlogs.time ignore_missing: true - rename: field: azure.signinlogs.resourceId @@ -32,6 +33,15 @@ processors: field: azure.signinlogs.callerIpAddress target_field: source.ip ignore_missing: true + - set: + field: client.ip + value: '{{source.ip}}' + ignore_empty_value: true + - append: + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + if: 'ctx.source?.ip != null' - rename: field: azure.signinlogs.Level target_field: log.level diff --git a/packages/azure/data_stream/signinlogs/fields/ecs.yml b/packages/azure/data_stream/signinlogs/fields/ecs.yml index 6ed7adae3a8..11efd619e1e 100644 --- a/packages/azure/data_stream/signinlogs/fields/ecs.yml +++ b/packages/azure/data_stream/signinlogs/fields/ecs.yml @@ -1,3 +1,6 @@ +- description: IP address of the client. + name: client.ip + type: ip - description: Destination network address. ignore_above: 1024 name: destination.address diff --git a/packages/azure/docs/README.md b/packages/azure/docs/README.md index e0893e0ceb1..e68e15e9653 100644 --- a/packages/azure/docs/README.md +++ b/packages/azure/docs/README.md @@ -210,6 +210,7 @@ An example event for `activitylogs` looks as following: | azure.resource.provider | Resource type/namespace | keyword | | azure.subscription_id | Azure subscription ID | keyword | | azure.tenant_id | tenant ID | keyword | +| client.ip | IP address of the client. | ip | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | @@ -843,6 +844,7 @@ An example event for `signinlogs` looks as following: | azure.signinlogs.tenant_id | Tenant ID | keyword | | azure.subscription_id | Azure subscription ID | keyword | | azure.tenant_id | tenant ID | keyword | +| client.ip | IP address of the client. | ip | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host is running. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | diff --git a/packages/azure/manifest.yml b/packages/azure/manifest.yml index d5686e9f796..f41ec799aef 100644 --- a/packages/azure/manifest.yml +++ b/packages/azure/manifest.yml @@ -1,6 +1,6 @@ name: azure title: Azure -version: 0.1.7 +version: 0.2.0 release: beta description: Azure Integration type: integration From 1eab9d57115bcae1c7fde0cf3c8b30fa4bfd3ce4 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 18 Feb 2021 09:43:30 +0100 Subject: [PATCH 2/2] Add changelog --- packages/azure/changelog.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml index 681566521fb..88ef1801486 100644 --- a/packages/azure/changelog.yml +++ b/packages/azure/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.2.0" + changes: + - description: Add changes to use ECS 1.8 fields. + type: enhancement # can be one of: enhancement, bugfix, breaking-change + link: https://github.com/elastic/integrations/pull/722 - version: "0.0.1" changes: - description: initial release