[ECS][sentinel_one] Correcting invalid ECS field usages at root-level (…

…#7985) * Correcting invalid ECS field usages at root-level * changelog
elastic · Sep 26, 2023 · 9e6f048 · 9e6f048
1 parent 3e15157
commit 9e6f048
Show file tree

Hide file tree

Showing 13 changed files with 98 additions and 95 deletions.
diff --git a/packages/sentinel_one/changelog.yml b/packages/sentinel_one/changelog.yml
@@ -1,5 +1,10 @@
 # newer versions go on top
-- version: 1.14.0
+- version: "1.15.0"
+  changes:
+    - description: Correct invalid ECS field usages at root-level.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/7985
+- version: "1.14.0"
   changes:
     - description: ECS version updated to 8.10.0.
       type: enhancement

diff --git a/...inel_one/data_stream/activity/_dev/test/pipeline/test-pipeline-activity.log-expected.json b/...inel_one/data_stream/activity/_dev/test/pipeline/test-pipeline-activity.log-expected.json
@@ -458,8 +458,10 @@
                     "family": "osname"
                 }
             },
-            "os": {
-                "family": "osname"
+            "observer": {
+                "os": {
+                    "family": "osname"
+                }
             },
             "process": {
                 "hash": {
@@ -913,8 +915,10 @@
                     "family": "linux"
                 }
             },
-            "os": {
-                "family": "linux"
+            "observer": {
+                "os": {
+                    "family": "linux"
+                }
             },
             "process": {
                 "hash": {
@@ -1429,8 +1433,10 @@
                     "family": "linux"
                 }
             },
-            "os": {
-                "family": "linux"
+            "observer": {
+                "os": {
+                    "family": "linux"
+                }
             },
             "process": {
                 "hash": {

diff --git a/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/activity/elasticsearch/ingest_pipeline/default.yml
@@ -113,7 +113,7 @@ processors:
       ignore_failure: true
   - rename:
       field: json.osFamily
-      target_field: os.family
+      target_field: observer.os.family
       ignore_missing: true
   - rename:
       field: json.agentUpdatedVersion

diff --git a/packages/sentinel_one/data_stream/activity/fields/ecs.yml b/packages/sentinel_one/data_stream/activity/fields/ecs.yml
@@ -35,7 +35,7 @@
 - external: ecs
   name: observer.version
 - external: ecs
-  name: os.family
+  name: observer.os.family
 - external: ecs
   name: process.hash.sha1
 - external: ecs

diff --git a/packages/sentinel_one/data_stream/activity/sample_event.json b/packages/sentinel_one/data_stream/activity/sample_event.json
@@ -1,12 +1,11 @@
 {
     "@timestamp": "2022-04-05T16:01:56.995Z",
     "agent": {
-        "ephemeral_id": "301c0e56-31de-4db5-a8d9-7ee30c9de5fb",
-        "hostname": "docker-fleet-agent",
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "ephemeral_id": "b33ff3ed-caeb-4ae9-91b6-587a31f52e85",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "data_stream": {
         "dataset": "sentinel_one.activity",
@@ -17,18 +16,18 @@
         "version": "8.10.0"
     },
     "elastic_agent": {
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "configuration"
         ],
-        "created": "2022-11-17T06:07:48.213Z",
+        "created": "2023-09-26T17:11:24.181Z",
         "dataset": "sentinel_one.activity",
-        "ingested": "2022-11-17T06:07:49Z",
+        "ingested": "2023-09-26T17:11:25Z",
         "kind": "event",
         "original": "{\"accountId\":\"1234567890123456789\",\"accountName\":\"Default\",\"activityType\":1234,\"agentId\":null,\"agentUpdatedVersion\":null,\"comments\":null,\"createdAt\":\"2022-04-05T16:01:56.995120Z\",\"data\":{\"accountId\":1234567890123456800,\"accountName\":\"Default\",\"fullScopeDetails\":\"Account Default\",\"fullScopeDetailsPath\":\"test/path\",\"groupName\":null,\"scopeLevel\":\"Account\",\"scopeName\":\"Default\",\"siteName\":null,\"username\":\"test user\"},\"description\":null,\"groupId\":null,\"groupName\":null,\"hash\":null,\"id\":\"1234567890123456789\",\"osFamily\":null,\"primaryDescription\":\"created Default account.\",\"secondaryDescription\":null,\"siteId\":null,\"siteName\":null,\"threatId\":null,\"updatedAt\":\"2022-04-05T16:01:56.992136Z\",\"userId\":\"1234567890123456789\"}",
         "type": [

diff --git a/packages/sentinel_one/data_stream/agent/sample_event.json b/packages/sentinel_one/data_stream/agent/sample_event.json
@@ -1,12 +1,11 @@
 {
     "@timestamp": "2022-04-07T08:31:47.481Z",
     "agent": {
-        "ephemeral_id": "29f3a613-970d-480f-942c-f568dd43da61",
-        "hostname": "docker-fleet-agent",
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "ephemeral_id": "a21f7739-b474-4166-9008-623141ee0c89",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "data_stream": {
         "dataset": "sentinel_one.agent",
@@ -17,18 +16,18 @@
         "version": "8.10.0"
     },
     "elastic_agent": {
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "host"
         ],
-        "created": "2022-11-17T06:08:29.695Z",
+        "created": "2023-09-26T17:12:30.343Z",
         "dataset": "sentinel_one.agent",
-        "ingested": "2022-11-17T06:08:30Z",
+        "ingested": "2023-09-26T17:12:33Z",
         "kind": "event",
         "original": "{\"accountId\":\"12345123451234512345\",\"accountName\":\"Account Name\",\"activeDirectory\":{\"computerDistinguishedName\":null,\"computerMemberOf\":[],\"lastUserDistinguishedName\":null,\"lastUserMemberOf\":[]},\"activeThreats\":7,\"agentVersion\":\"12.x.x.x\",\"allowRemoteShell\":true,\"appsVulnerabilityStatus\":\"not_applicable\",\"cloudProviders\":{},\"computerName\":\"user-test\",\"consoleMigrationStatus\":\"N/A\",\"coreCount\":2,\"cpuCount\":2,\"cpuId\":\"CPU Name\",\"createdAt\":\"2022-03-18T09:12:00.519500Z\",\"detectionState\":null,\"domain\":\"WORKGROUP\",\"encryptedApplications\":false,\"externalId\":\"\",\"externalIp\":\"81.2.69.143\",\"firewallEnabled\":true,\"firstFullModeTime\":null,\"groupId\":\"1234567890123456789\",\"groupIp\":\"81.2.69.144\",\"groupName\":\"Default Group\",\"id\":\"13491234512345\",\"inRemoteShellSession\":false,\"infected\":true,\"installerType\":\".msi\",\"isActive\":true,\"isDecommissioned\":false,\"isPendingUninstall\":false,\"isUninstalled\":false,\"isUpToDate\":true,\"lastActiveDate\":\"2022-03-17T09:51:28.506000Z\",\"lastIpToMgmt\":\"81.2.69.145\",\"lastLoggedInUserName\":\"\",\"licenseKey\":\"\",\"locationEnabled\":true,\"locationType\":\"not_applicable\",\"locations\":null,\"machineType\":\"server\",\"mitigationMode\":\"detect\",\"mitigationModeSuspicious\":\"detect\",\"modelName\":\"Compute Engine\",\"networkInterfaces\":[{\"gatewayIp\":\"81.2.69.145\",\"gatewayMacAddress\":\"00-00-5E-00-53-00\",\"id\":\"1234567890123456789\",\"inet\":[\"81.2.69.144\"],\"inet6\":[\"2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\"],\"name\":\"Ethernet\",\"physical\":\"00-00-5E-00-53-00\"}],\"networkQuarantineEnabled\":false,\"networkStatus\":\"connected\",\"operationalState\":\"na\",\"operationalStateExpiration\":null,\"osArch\":\"64 bit\",\"osName\":\"Linux Server\",\"osRevision\":\"1234\",\"osStartTime\":\"2022-04-06T08:27:14Z\",\"osType\":\"linux\",\"osUsername\":null,\"rangerStatus\":\"Enabled\",\"rangerVersion\":\"21.x.x.x\",\"registeredAt\":\"2022-04-06T08:26:45.515278Z\",\"remoteProfilingState\":\"disabled\",\"remoteProfilingStateExpiration\":null,\"scanAbortedAt\":null,\"scanFinishedAt\":\"2022-04-06T09:18:21.090855Z\",\"scanStartedAt\":\"2022-04-06T08:26:52.838047Z\",\"scanStatus\":\"finished\",\"siteId\":\"1234567890123456789\",\"siteName\":\"Default site\",\"storageName\":null,\"storageType\":null,\"tags\":{\"sentinelone\":[{\"assignedAt\":\"2018-02-27T04:49:26.257525Z\",\"assignedBy\":\"test-user\",\"assignedById\":\"123456789012345678\",\"id\":\"123456789012345678\",\"key\":\"key123\",\"value\":\"value123\"}]},\"threatRebootRequired\":false,\"totalMemory\":1234,\"updatedAt\":\"2022-04-07T08:31:47.481227Z\",\"userActionsNeeded\":[\"reboot_needed\"],\"uuid\":\"XXX35XXX8Xfb4aX0X1X8X12X343X8X30\"}",
         "type": [

diff --git a/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml b/packages/sentinel_one/data_stream/alert/elasticsearch/ingest_pipeline/default.yml
@@ -359,7 +359,7 @@ processors:
       ignore_missing: true
   - rename:
       field: json.osName
-      target_field: os.name
+      target_field: observer.os.name
       ignore_missing: true
   - rename:
       field: json.ruleInfo.type

diff --git a/packages/sentinel_one/data_stream/alert/fields/ecs.yml b/packages/sentinel_one/data_stream/alert/fields/ecs.yml
@@ -37,7 +37,7 @@
 - external: ecs
   name: orchestrator.namespace
 - external: ecs
-  name: os.name
+  name: observer.os.name
 - external: ecs
   name: process.code_signature.signing_id
 - external: ecs

diff --git a/packages/sentinel_one/data_stream/alert/sample_event.json b/packages/sentinel_one/data_stream/alert/sample_event.json
@@ -1,12 +1,11 @@
 {
     "@timestamp": "2018-02-27T04:49:26.257Z",
     "agent": {
-        "ephemeral_id": "304690e0-fe34-46e6-8f72-4d8b1085c1bb",
-        "hostname": "docker-fleet-agent",
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "ephemeral_id": "4172dd9c-9551-46b4-ae0d-c9faf59b7225",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "container": {
         "id": "string",
@@ -39,19 +38,19 @@
         "version": "8.10.0"
     },
     "elastic_agent": {
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "malware"
         ],
-        "created": "2022-11-17T06:09:11.447Z",
+        "created": "2023-09-26T17:13:36.438Z",
         "dataset": "sentinel_one.alert",
         "id": "123456789123456789",
-        "ingested": "2022-11-17T06:09:12Z",
+        "ingested": "2023-09-26T17:13:39Z",
         "kind": "event",
         "original": "{\"agentDetectionInfo\":{\"machineType\":\"string\",\"name\":\"string\",\"osFamily\":\"string\",\"osName\":\"string\",\"osRevision\":\"string\",\"siteId\":\"123456789123456789\",\"uuid\":\"string\",\"version\":\"3.x.x.x\"},\"alertInfo\":{\"alertId\":\"123456789123456789\",\"analystVerdict\":\"string\",\"createdAt\":\"2018-02-27T04:49:26.257525Z\",\"dnsRequest\":\"string\",\"dnsResponse\":\"string\",\"dstIp\":\"81.2.69.144\",\"dstPort\":\"1234\",\"dvEventId\":\"string\",\"eventType\":\"info\",\"hitType\":\"Events\",\"incidentStatus\":\"string\",\"indicatorCategory\":\"string\",\"indicatorDescription\":\"string\",\"indicatorName\":\"string\",\"loginAccountDomain\":\"string\",\"loginAccountSid\":\"string\",\"loginIsAdministratorEquivalent\":\"string\",\"loginIsSuccessful\":\"string\",\"loginType\":\"string\",\"loginsUserName\":\"string\",\"modulePath\":\"string\",\"moduleSha1\":\"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d\",\"netEventDirection\":\"string\",\"registryKeyPath\":\"string\",\"registryOldValue\":\"string\",\"registryOldValueType\":\"string\",\"registryPath\":\"string\",\"registryValue\":\"string\",\"reportedAt\":\"2018-02-27T04:49:26.257525Z\",\"source\":\"string\",\"srcIp\":\"81.2.69.142\",\"srcMachineIp\":\"81.2.69.142\",\"srcPort\":\"1234\",\"tiIndicatorComparisonMethod\":\"string\",\"tiIndicatorSource\":\"string\",\"tiIndicatorType\":\"string\",\"tiIndicatorValue\":\"string\",\"updatedAt\":\"2018-02-27T04:49:26.257525Z\"},\"containerInfo\":{\"id\":\"string\",\"image\":\"string\",\"labels\":\"string\",\"name\":\"string\"},\"kubernetesInfo\":{\"cluster\":\"string\",\"controllerKind\":\"string\",\"controllerLabels\":\"string\",\"controllerName\":\"string\",\"namespace\":\"string\",\"namespaceLabels\":\"string\",\"node\":\"string\",\"pod\":\"string\",\"podLabels\":\"string\"},\"ruleInfo\":{\"description\":\"string\",\"id\":\"string\",\"name\":\"string\",\"scopeLevel\":\"string\",\"severity\":\"Low\",\"treatAsThreat\":\"UNDEFINED\"},\"sourceParentProcessInfo\":{\"commandline\":\"string\",\"fileHashMd5\":\"5d41402abc4b2a76b9719d911017c592\",\"fileHashSha1\":\"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d\",\"fileHashSha256\":\"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\",\"filePath\":\"string\",\"fileSignerIdentity\":\"string\",\"integrityLevel\":\"unknown\",\"name\":\"string\",\"pid\":\"12345\",\"pidStarttime\":\"2018-02-27T04:49:26.257525Z\",\"storyline\":\"string\",\"subsystem\":\"unknown\",\"uniqueId\":\"string\",\"user\":\"string\"},\"sourceProcessInfo\":{\"commandline\":\"string\",\"fileHashMd5\":\"5d41402abc4b2a76b9719d911017c592\",\"fileHashSha1\":\"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d\",\"fileHashSha256\":\"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\",\"filePath\":\"string\",\"fileSignerIdentity\":\"string\",\"integrityLevel\":\"unknown\",\"name\":\"string\",\"pid\":\"12345\",\"pidStarttime\":\"2018-02-27T04:49:26.257525Z\",\"storyline\":\"string\",\"subsystem\":\"unknown\",\"uniqueId\":\"string\",\"user\":\"string\"},\"targetProcessInfo\":{\"tgtFileCreatedAt\":\"2018-02-27T04:49:26.257525Z\",\"tgtFileHashSha1\":\"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d\",\"tgtFileHashSha256\":\"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824\",\"tgtFileId\":\"string\",\"tgtFileIsSigned\":\"string\",\"tgtFileModifiedAt\":\"2018-02-27T04:49:26.257525Z\",\"tgtFileOldPath\":\"string\",\"tgtFilePath\":\"string\",\"tgtProcCmdLine\":\"string\",\"tgtProcImagePath\":\"string\",\"tgtProcIntegrityLevel\":\"unknown\",\"tgtProcName\":\"string\",\"tgtProcPid\":\"12345\",\"tgtProcSignedStatus\":\"string\",\"tgtProcStorylineId\":\"string\",\"tgtProcUid\":\"string\",\"tgtProcessStartTime\":\"2018-02-27T04:49:26.257525Z\"}}",
         "type": [

diff --git a/packages/sentinel_one/data_stream/group/sample_event.json b/packages/sentinel_one/data_stream/group/sample_event.json
@@ -1,12 +1,11 @@
 {
     "@timestamp": "2022-04-05T16:01:57.564Z",
     "agent": {
-        "ephemeral_id": "fb84b4b0-03b8-437d-b616-587370d25d48",
-        "hostname": "docker-fleet-agent",
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "ephemeral_id": "cedb118a-3e1f-412e-8f10-4e566d7b81fa",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "data_stream": {
         "dataset": "sentinel_one.group",
@@ -17,18 +16,18 @@
         "version": "8.10.0"
     },
     "elastic_agent": {
-        "id": "b87eb683-6c5a-4db7-86d4-96694d38752b",
+        "id": "f25d13cd-18cc-4e73-822c-c4f849322623",
         "snapshot": false,
-        "version": "7.17.0"
+        "version": "8.10.1"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "iam"
         ],
-        "created": "2022-11-17T06:09:54.643Z",
+        "created": "2023-09-26T17:14:41.625Z",
         "dataset": "sentinel_one.group",
-        "ingested": "2022-11-17T06:09:55Z",
+        "ingested": "2023-09-26T17:14:44Z",
         "kind": "event",
         "original": "{\"createdAt\":\"2022-04-05T16:01:56.928383Z\",\"creator\":\"Test User\",\"creatorId\":\"1234567890123456789\",\"filterId\":null,\"filterName\":null,\"id\":\"1234567890123456789\",\"inherits\":true,\"isDefault\":true,\"name\":\"Default Group\",\"rank\":null,\"registrationToken\":\"eyxxxxxxxxxxxxxxxxxxxxkixZxx1xxxxx8xxx2xODA0ZxxxxTIwNjhxxxxxxxxxxxxxxiMWYxx1Ixxnxxxx0=\",\"siteId\":\"1234567890123456789\",\"totalAgents\":1,\"type\":\"static\",\"updatedAt\":\"2022-04-05T16:01:57.564266Z\"}",
         "type": [