[8.x] [SecuritySolution] Service Entity Store (#202344) (#203521)

# Backport This will backport the following commits from `main` to `8.x`: - [[SecuritySolution] Service Entity Store (#202344)](#202344)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport)
elastic · Dec 10, 2024 · 219736a · 219736a
1 parent f207abf
commit 219736a
Show file tree

Hide file tree

Showing 16 changed files with 416 additions and 20 deletions.
diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -41033,6 +41033,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object

diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type EntityType = z.infer<typeof EntityType>;
-export const EntityType = z.enum(['user', 'host']);
+export const EntityType = z.enum(['user', 'host', 'service']);
 export type EntityTypeEnum = typeof EntityType.enum;
 export const EntityTypeEnum = EntityType.enum;
 

diff --git a/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -11,6 +11,7 @@ components:
       enum:
         - user
         - host
+        - service
 
     EngineDescriptor:
       type: object

diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts
@@ -243,6 +243,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   entityStoreDisabled: false,
 
+  /**
+   * Enables the Service Entity Store. The Entity Store feature will install the service engine by default.
+   */
+  serviceEntityStoreEnabled: true,
+
   /**
    * Enables the siem migrations feature
    */

diff --git a/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/...on/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -972,6 +972,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

diff --git a/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml b/.../openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml
@@ -972,6 +972,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

diff --git a/...curity_solution/public/entity_analytics/components/entity_store/hooks/use_entity_store.ts b/...curity_solution/public/entity_analytics/components/entity_store/hooks/use_entity_store.ts
@@ -12,10 +12,11 @@ import type { IHttpFetchError } from '@kbn/core-http-browser';
 import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
 import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { useKibana } from '../../../../common/lib/kibana/kibana_react';
-import type {
-  DeleteEntityEngineResponse,
-  InitEntityEngineResponse,
-  StopEntityEngineResponse,
+import type { EntityType } from '../../../../../common/api/entity_analytics';
+import {
+  type DeleteEntityEngineResponse,
+  type InitEntityEngineResponse,
+  type StopEntityEngineResponse,
 } from '../../../../../common/api/entity_analytics';
 import { useEntityStoreRoutes } from '../../../api/entity_store';
 import { EntityEventTypes } from '../../../../common/lib/telemetry';
@@ -68,13 +69,16 @@ export const useEnableEntityStoreMutation = (options?: UseMutationOptions<{}>) =
 };
 
 export const INIT_ENTITY_ENGINE_STATUS_KEY = ['POST', 'INIT_ENTITY_ENGINE'];
+/**
+ * @deprecated
+ * It will be deleted on a follow-up PR
+ */
 export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
   const queryClient = useQueryClient();
 
   const { initEntityEngine } = useEntityStoreRoutes();
   return useMutation<InitEntityEngineResponse[]>(
     () => Promise.all([initEntityEngine('user'), initEntityEngine('host')]),
-
     {
       mutationKey: INIT_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
@@ -84,7 +88,7 @@ export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
 };
 
 export const STOP_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
+export const useStopEntityEngineMutation = (entityTypes: EntityType[]) => {
   const { telemetry } = useKibana().services;
   const queryClient = useQueryClient();
 
@@ -95,23 +99,28 @@ export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
         timestamp: new Date().toISOString(),
         action: 'stop',
       });
-      return Promise.all([stopEntityEngine('user'), stopEntityEngine('host')]);
+      return Promise.all(entityTypes.map((entityType) => stopEntityEngine(entityType)));
     },
     {
       mutationKey: STOP_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
-      ...options,
     }
   );
 };
 
 export const DELETE_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useDeleteEntityEngineMutation = ({ onSuccess }: { onSuccess?: () => void }) => {
+export const useDeleteEntityEngineMutation = ({
+  onSuccess,
+  entityTypes,
+}: {
+  onSuccess?: () => void;
+  entityTypes: EntityType[];
+}) => {
   const queryClient = useQueryClient();
   const { deleteEntityEngine } = useEntityStoreRoutes();
 
   return useMutation<DeleteEntityEngineResponse[]>(
-    () => Promise.all([deleteEntityEngine('user', true), deleteEntityEngine('host', true)]),
+    () => Promise.all(entityTypes.map((entityType) => deleteEntityEngine(entityType, true))),
     {
       mutationKey: DELETE_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => {

diff --git a/.../plugins/security_solution/public/entity_analytics/pages/entity_store_management_page.tsx b/.../plugins/security_solution/public/entity_analytics/pages/entity_store_management_page.tsx
@@ -33,7 +33,7 @@ import React, { useCallback, useEffect, useState } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 
 import type { SecurityAppError } from '@kbn/securitysolution-t-grid';
-import type { StoreStatus } from '../../../common/api/entity_analytics';
+import { EntityType, EntityTypeEnum, type StoreStatus } from '../../../common/api/entity_analytics';
 import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
 import { ASSET_CRITICALITY_INDEX_PATTERN } from '../../../common/entity_analytics/asset_criticality';
 import { useKibana } from '../../common/lib/kibana';
@@ -73,13 +73,20 @@ export const EntityStoreManagementPage = () => {
   const hasAssetCriticalityWritePermissions = assetCriticalityPrivileges?.has_write_permissions;
   const [selectedTabId, setSelectedTabId] = useState(TabId.Import);
   const entityStoreStatus = useEntityStoreStatus({});
+  const isServiceEntityStoreEnabled = useIsExperimentalFeatureEnabled('serviceEntityStoreEnabled');
+  const allEntityTypes = Object.values(EntityType.Values);
+
+  const entityTypes = isServiceEntityStoreEnabled
+    ? allEntityTypes
+    : allEntityTypes.filter((value) => value !== EntityTypeEnum.service);
 
   const enableStoreMutation = useEnableEntityStoreMutation();
-  const stopEntityEngineMutation = useStopEntityEngineMutation();
+  const stopEntityEngineMutation = useStopEntityEngineMutation(entityTypes);
   const deleteEntityEngineMutation = useDeleteEntityEngineMutation({
     onSuccess: () => {
       closeClearModal();
     },
+    entityTypes,
   });
 
   const [isClearModalVisible, setIsClearModalVisible] = useState(false);

diff --git a/...curity_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts b/...curity_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts
@@ -16,6 +16,7 @@ import type { EntityType } from '../../../../common/api/entity_analytics/entity_
 import type { DataViewsService } from '@kbn/data-views-plugin/common';
 import type { AppClient } from '../../..';
 import type { EntityStoreConfig } from './types';
+import { mockGlobalState } from '../../../../public/common/mock';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
@@ -31,6 +32,7 @@ describe('EntityStoreDataClient', () => {
     dataViewsService: {} as DataViewsService,
     appClient: {} as AppClient,
     config: {} as EntityStoreConfig,
+    experimentalFeatures: mockGlobalState.app.enableExperimental,
   });
 
   const defaultSearchParams = {

diff --git a/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts b/...ns/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts
@@ -23,6 +23,7 @@ import moment from 'moment';
 import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import type { estypes } from '@elastic/elasticsearch';
+import type { ExperimentalFeatures } from '../../../../common';
 import type {
   GetEntityStoreStatusRequestQuery,
   GetEntityStoreStatusResponse,
@@ -32,7 +33,10 @@ import type {
   InitEntityStoreResponse,
 } from '../../../../common/api/entity_analytics/entity_store/enable.gen';
 import type { AppClient } from '../../..';
-import { EngineComponentResourceEnum, EntityType } from '../../../../common/api/entity_analytics';
+import {
+  EngineComponentResourceEnum,
+  EntityTypeEnum,
+} from '../../../../common/api/entity_analytics';
 import type {
   Entity,
   EngineDataviewUpdateResult,
@@ -42,6 +46,7 @@ import type {
   ListEntityEnginesResponse,
   EngineComponentStatus,
   EngineComponentResource,
+  EntityType,
 } from '../../../../common/api/entity_analytics';
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
 import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
@@ -108,6 +113,7 @@ interface EntityStoreClientOpts {
   kibanaVersion: string;
   dataViewsService: DataViewsService;
   appClient: AppClient;
+  experimentalFeatures: ExperimentalFeatures;
   telemetry?: AnalyticsServiceSetup;
   config: EntityStoreConfig;
 }
@@ -205,7 +211,13 @@ export class EntityStoreDataClient {
     // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
     const run = <T>(fn: () => Promise<T>) =>
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
-    const promises = Object.values(EntityType.Values).map((entity) =>
+
+    const { experimentalFeatures } = this.options;
+    const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
+      ? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
+      : [EntityTypeEnum.host, EntityTypeEnum.user];
+
+    const promises = enginesTypes.map((entity) =>
       run(() =>
         this.init(entity, { indexPattern, filter, fieldHistoryLength }, { pipelineDebugMode })
       )

diff --git a/.../server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts b/.../server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts
@@ -7,4 +7,5 @@
 
 export * from './host';
 export * from './user';
+export * from './service';
 export { getCommonUnitedFieldDefinitions } from './common';
diff --git a/...erver/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts b/...erver/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts
@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { collectValuesWithLength, newestValue } from '../definition_utils';
+import type { UnitedDefinitionBuilder } from '../types';
+
+export const SERVICE_DEFINITION_VERSION = '1.0.0';
+export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
+  const collect = collectValuesWithLength(fieldHistoryLength);
+  return {
+    entityType: 'service',
+    version: SERVICE_DEFINITION_VERSION,
+    fields: [
+      collect({ field: 'service.address' }),
+      collect({ field: 'service.environment' }),
+      collect({ field: 'service.ephemeral_id' }),
+      collect({ field: 'service.id' }),
+      collect({ field: 'service.node.name' }),
+      collect({ field: 'service.node.roles' }),
+      newestValue({ field: 'service.state' }),
+      collect({ field: 'service.type' }),
+      newestValue({ field: 'service.version' }),
+    ],
+  };
+};
-Original file line number
+Diff line change
@@ Expand Up / @@ -41033,6 +41033,7 @@ components: @@
           enum:
             - user
             - host
+            - service
           type: string
         Security_Entity_Analytics_API_HostEntity:
           type: object
@@ Expand Down @@