elastic · marc-gr · Feb 18, 2021 · Feb 17, 2021 · Feb 18, 2021
diff --git a/packages/azure/changelog.yml b/packages/azure/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.2.0"
+  changes:
+    - description: Add changes to use ECS 1.8 fields.
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/722
 - version: "0.0.1"
   changes:
     - description: initial release

diff --git a/...azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json b/...azure/data_stream/activitylogs/_dev/test/pipeline/test-activitylogs-raw.log-expected.json
@@ -1,6 +1,21 @@
 {
     "expected": [
         {
+            "log": {
+                "level": "Information"
+            },
+            "source": {
+                "geo": {
+                    "continent_name": "Europe",
+                    "country_name": "United Kingdom",
+                    "location": {
+                        "lon": -0.1224,
+                        "lat": 51.4964
+                    },
+                    "country_iso_code": "GB"
+                },
+                "ip": "51.251.141.41"
+            },
             "geo": {
                 "continent_name": "Europe",
                 "country_name": "United Kingdom",
@@ -14,28 +29,22 @@
                 "provider": "azure"
             },
             "@timestamp": "2019-10-24T00:13:46.355Z",
+            "related": {
+                "ip": [
+                    "51.251.141.41"
+                ]
+            },
             "ecs": {
                 "version": "1.5.0"
             },
-            "log": {
-                "level": "Information"
-            },
-            "source": {
-                "geo": {
-                    "continent_name": "Europe",
-                    "country_name": "United Kingdom",
-                    "location": {
-                        "lon": -0.1224,
-                        "lat": 51.4964
-                    },
-                    "country_iso_code": "GB"
-                },
+            "client": {
                 "ip": "51.251.141.41"
             },
             "event": {
                 "duration": 0,
                 "action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION",
-                "ingested": "2020-12-04T13:40:57.414565400Z",
+                "ingested": "2021-02-17T15:44:41.246811100Z",
+                "original": "{\"callerIpAddress\":\"51.251.141.41\",\"category\":\"Action\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":{\"authorization\":{\"action\":\"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action\",\"evidence\":{\"principalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"principalType\":\"ServicePrincipal\",\"role\":\"Azure EventGrid Service BuiltIn Role\",\"roleAssignmentId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleAssignmentScope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"roleDefinitionId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\"},\"scope\":\"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey\"},\"claims\":{\"aio\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"appidacr\":\"2\",\"aud\":\"https://management.core.windows.net/\",\"exp\":\"1571904826\",\"http://schemas.microsoft.com/identity/claims/identityprovider\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"http://schemas.microsoft.com/identity/claims/objectidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.microsoft.com/identity/claims/tenantid\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"iat\":\"1571875726\",\"iss\":\"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/\",\"nbf\":\"1571875726\",\"uti\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"ver\":\"1.0\"}},\"level\":\"Information\",\"location\":\"global\",\"operationName\":\"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION\",\"resourceId\":\"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY\",\"resultSignature\":\"Started.\",\"resultType\":\"Start\",\"time\":\"2019-10-24T00:13:46.3554259Z\"}",
                 "type": [
                     "change"
                 ],

diff --git a/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/azure-eventhub.yml.hbs
@@ -11,4 +11,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/activitylogs/agent/stream/log.yml.hbs
@@ -9,4 +9,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml
@@ -22,10 +22,11 @@ processors:
       ignore_failure: true
       formats:
         - ISO8601
+  - rename:
+      field: message
+      target_field: event.original
   - remove:
-      field:
-        - message
-        - azure.activitylogs.time
+      field: azure.activitylogs.time
       ignore_missing: true
   - rename:
       field: azure.activitylogs.resourceId
@@ -35,6 +36,15 @@ processors:
       field: azure.activitylogs.callerIpAddress
       target_field: source.ip
       ignore_missing: true
+  - set:
+      field: client.ip
+      value: '{{source.ip}}'
+      ignore_empty_value: true
+  - append:
+      field: related.ip
+      value: '{{source.ip}}'
+      allow_duplicates: false
+      if: 'ctx.source?.ip != null'
   - rename:
       field: azure.activitylogs.level
       target_field: log.level
@@ -224,6 +234,26 @@ processors:
       patterns:
         - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}'
       ignore_missing: true
+      ignore_failure: true
+
+  # set user.email to the original name if the above grok succeeded.
+  - set:
+      field: user.email
+      value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}'
+      ignore_empty_value: true
+      if: 'ctx.user?.name != null'
+
+  # set user.name to the original name if the above grok failed (name format is not an email).
+  - set:
+      field: user.name
+      value: '{{azure.activitylogs.identity.claims_initiated_by_user.name}}'
+      ignore_empty_value: true
+      if: 'ctx.user?.name == null'
+  - append:
+      field: related.user
+      value: '{{user.name}}'
+      allow_duplicates: false
+      if: 'ctx.user?.name != null'
   - convert:
       field: azure.activitylogs.identity.claims_initiated_by_user.fullname
       target_field: user.full_name

diff --git a/packages/azure/data_stream/activitylogs/fields/ecs.yml b/packages/azure/data_stream/activitylogs/fields/ecs.yml
@@ -1,3 +1,6 @@
+- description: IP address of the client.
+  name: client.ip
+  type: ip
 - description: Destination network address.
   ignore_above: 1024
   name: destination.address

diff --git a/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json b/packages/azure/data_stream/auditlogs/_dev/test/pipeline/test-auditlogs-raw.log-expected.json
@@ -14,7 +14,8 @@
             "event": {
                 "duration": 0,
                 "action": "Update device",
-                "ingested": "2020-12-04T13:40:57.990898700Z",
+                "ingested": "2021-02-17T15:44:41.925301300Z",
+                "original": "{\"category\":\"AuditLogs\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"durationMs\":0,\"identity\":\"Device Registration Service\",\"level\":\"Informational\",\"operationName\":\"Update device\",\"operationVersion\":\"1.0\",\"properties\":{\"activityDateTime\":\"2019-10-18T15:30:51.0273716+00:00\",\"activityDisplayName\":\"Update device\",\"category\":\"Device\",\"correlationId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"id\":\"Directory_ESQ\",\"initiatedBy\":{\"app\":{\"appId\":\"id\",\"displayName\":\"Device Registration Service\",\"servicePrincipalId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"servicePrincipalName\":\"Core\"}},\"loggedByService\":\"Core Directory\",\"operationType\":\"Update\",\"result\":\"success\",\"resultReason\":\"\",\"targetResources\":[{\"displayName\":\"LAPTOP-12\",\"id\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"modifiedProperties\":[{\"displayName\":\"Included Updated Properties\",\"newValue\":\"\\\"\\\"\",\"oldValue\":\"\"}],\"type\":\"Device\"}]},\"resourceId\":\"/tenants/8a4de8b5-095c-47d0-a96f-a75130c61d53/providers/Microsoft.aadiam\",\"resultSignature\":\"None\",\"tenantId\":\"8a4de8b5-095c-47d0-a96f-a75130c61d53\",\"time\":\"2019-10-18T15:30:51.0273716Z\"}",
                 "kind": "event",
                 "outcome": "success"
             },

diff --git a/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/azure-eventhub.yml.hbs
@@ -11,4 +11,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/auditlogs/agent/stream/log.yml.hbs
@@ -9,4 +9,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml
@@ -40,10 +40,11 @@ processors:
       field: azure.auditlogs.level
       target_field: log.level
       ignore_missing: true
+  - rename:
+      field: message
+      target_field: event.original
   - remove:
-      field:
-        - message
-        - azure.auditlogs.time
+      field: azure.auditlogs.time
       ignore_missing: true
   - convert:
       field: azure.auditlogs.operationName

diff --git a/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json b/...ta_stream/platformlogs/_dev/test/pipeline/test-platformlogs-invalid-raw.log-expected.json
@@ -1,28 +1,12 @@
 {
     "expected": [
         {
-            "cloud": {
-                "provider": "azure"
-            },
-            "ecs": {
-                "version": "1.5.0"
-            },
             "message": "{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18234,\"httpMethod\":\"GET\",\"requestUri\":\"/nmaplowercheck1602448229\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":108,\"sentBytes\":1636,\"timeTaken\":78,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}},{ \"resourceId\": \"/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY\", \"operationName\": \"ApplicationGatewayAccess\", \"time\": \"2020-10-11T20:30:59Z\", \"category\": \"ApplicationGatewayAccessLog\", \"properties\": {\"instanceId\":\"ApplicationGatewayRole_IN_1\",\"clientIP\":\"172.105.13.165\",\"clientPort\":18706,\"httpMethod\":\"GET\",\"requestUri\":\"/evox/about\",\"requestQuery\":\"X-AzureApplicationGateway-CACHE-HIT=0\",\"userAgent\":\"\"Mozilla/5.0\",\"httpStatus\":502,\"httpVersion\":\"HTTP/1.1\",\"receivedBytes\":94,\"sentBytes\":1636,\"timeTaken\":62,\"sslEnabled\":\"off\",\"host\":\"IP\",\"originalHost\":\"IP\"}}]}",
             "event": {
-                "ingested": "2020-12-04T13:40:58.273944800Z",
-                "kind": "event"
+                "ingested": "2021-02-17T15:44:42.249938900Z"
             },
             "error": {
-                "message": "invalid json log"
-            },
-            "azure": {
-                "resource": {
-                    "id": "/SUBSCRIPTIONS/SUBSCRIPTION/RESOURCEGROUPS/ET-AZURE-INTEGRATION-TESTING/PROVIDERS/MICROSOFT.NETWORK/APPLICATIONGATEWAYS/LANGERGATEWAY"
-                },
-                "platformlogs": {
-                    "category": "ApplicationGatewayAccessLog",
-                    "event_category": "Administrative"
-                }
+                "message": "Unexpected character ('M' (code 77)): was expecting comma to separate Object entries\\n at [Source: (org.elasticsearch.common.io.stream.InputStreamStreamInput); line: 1, column: 509]"
             }
         }
     ]

diff --git a/...azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json b/...azure/data_stream/platformlogs/_dev/test/pipeline/test-platformlogs-raw.log-expected.json
@@ -11,7 +11,8 @@
             },
             "event": {
                 "action": "Retreive ConsumerGroup",
-                "ingested": "2020-12-04T13:40:58.303070600Z",
+                "ingested": "2021-02-17T15:44:42.262500900Z",
+                "original": "{\"ActivityId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8\",\"Caller\":\"Portal\",\"Environment\":\"PROD\",\"EventName\":\"Retreive ConsumerGroup\",\"EventProperties\":\"{\\\"SubscriptionId\\\":\\\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\\\",\\\"Namespace\\\":\\\"obstesteventhubs\\\",\\\"Via\\\":\\\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\\u0026$skip=0\\u0026$top=100\\\",\\\"TrackingId\\\":\\\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\\\"}\",\"EventTimeString\":\"11/3/2020 9:06:42 AM +00:00\",\"Region\":\"West Europe\",\"ScaleUnit\":\"PROD-AM3-AZ501\",\"Status\":\"Succeeded\",\"category\":\"OperationalLogs\",\"resourceId\":\"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS\"}",
                 "kind": "event",
                 "outcome": "succeeded"
             },

diff --git a/...ata_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json b/...ata_stream/platformlogs/_dev/test/pipeline/test-platformlogs-remote-raw.log-expected.json
@@ -10,7 +10,8 @@
             },
             "event": {
                 "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read",
-                "ingested": "2020-12-04T13:40:58.343665Z",
+                "ingested": "2021-02-17T15:44:42.305137300Z",
+                "original": "{\"Cloud\":\"AzureCloud\",\"Environment\":\"prod\",\"category\":\"kube-audit\",\"ccpNamespace\":\"5e4bf4baee195b00017cdbfa\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\\\"}\",\"pod\":\"kube-apiserver-666bd4b459-hjgdc\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE\",\"time\":\"2020-11-09T10:57:31.0000000Z\"}",
                 "kind": "event"
             },
             "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}",

diff --git a/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/azure-eventhub.yml.hbs
@@ -11,4 +11,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs b/packages/azure/data_stream/platformlogs/agent/stream/log.yml.hbs
@@ -9,4 +9,9 @@ tags:
 {{/each}}
 {{#contains tags "forwarded"}}
 publisher_pipeline.disable_host: true
-{{/contains}}
+{{/contains}}
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.8.0
diff --git a/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml b/packages/azure/data_stream/platformlogs/elasticsearch/ingest_pipeline/default.yml
@@ -16,23 +16,6 @@ processors:
   - json:
       field: message
       target_field: azure.platformlogs
-      on_failure:
-        - grok:
-            field: message
-            patterns:
-              - "resourceId\": \"%{DATA:azure.platformlogs.resourceId}\""
-            ignore_failure: true
-            ignore_missing: true
-        - grok:
-            field: message
-            patterns:
-              - "category\": \"%{DATA:azure.platformlogs.category}\""
-            ignore_failure: true
-            ignore_missing: true
-        - set:
-            field: error.message
-            value: 'invalid json log'
-            ignore_failure: true
   - date:
       field: azure.platformlogs.time
       target_field: '@timestamp'
@@ -46,14 +29,11 @@ processors:
       formats:
         - ISO8601
         - "M/d/yyyy h:mm:ss a XXX"
+  - rename:
+      field: message
+      target_field: event.original
   - remove:
-      if: "ctx.error?.message != 'invalid json log'"
-      field:
-        - message
-      ignore_missing: true
-  - remove:
-      field:
-        - azure.platformlogs.time
+      field: azure.platformlogs.time
       ignore_missing: true
   - rename:
       field: azure.platformlogs.resourceId
@@ -84,6 +64,15 @@ processors:
       field: azure.platformlogs.callerIpAddress
       target_field: source.ip
       ignore_missing: true
+  - set:
+      field: client.ip
+      value: '{{source.ip}}'
+      ignore_empty_value: true
+  - append:
+      field: related.ip
+      value: '{{source.ip}}'
+      allow_duplicates: false
+      if: 'ctx.source?.ip != null'
   - rename:
       field: azure.platformlogs.level
       target_field: log.level
@@ -124,19 +113,16 @@ processors:
       field: azure.platformlogs.result_type
       target_field: event.outcome
       type: string
-      ignore_missing: true
       if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')"
   - convert:
       field: azure.platformlogs.properties.result
       target_field: event.outcome
       type: string
       if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)"
-      ignore_missing: true
   - convert:
       field: azure.platformlogs.Status
       target_field: event.outcome
       type: string
-      ignore_missing: true
       if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)"
   - rename:
       field: azure.platformlogs.operationName
@@ -212,11 +198,9 @@ processors:
   - set:
       field: event.kind
       value: event
-      ignore_failure: true
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
 on_failure:
-  - set:
-      field: error.message
-      value: '{{ _ingest.on_failure_message }}'
-      ignore_failure: true
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'