[Entity Analytics] [Entity Store] Refactor entity store enablement (s…

…erver side) (elastic#199638)

## Summary

This PR adds 2 new endpoints regarding enablement of the Entity Store:
* `api/entity_store/enable`, which initializes entity engines for both
`user` and `host` entities
* `api/entity_store/status`, which computes a global store status based
on the individual engine status

In addition, running initialization of multiple engines in parallel is
now allowed.


### How to test

1. Use dev tools to call `POST kbn:/api/entity_store/enable`
2. Check that two engines were created and that the status is
`installing` by calling `GET kbn:/api/entity_store/status`
3. Wait a few seconds and keep calling the `status` endpoint. Once
initialization finishes, the status should switch to `running`

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

oas_docs/output/kibana.serverless.yaml

@@ -7395,6 +7395,43 @@ paths:
       tags:
         - Security Endpoint Management API
       x-beta: true
+  /api/entity_store/enable:
+    post:
+      operationId: InitEntityStore
+      requestBody:
+        content:
+          application/json; Elastic-Api-Version=2023-10-31:
+            schema:
+              type: object
+              properties:
+                fieldHistoryLength:
+                  default: 10
+                  description: The number of historical values to keep for each field.
+                  type: integer
+                filter:
+                  type: string
+                indexPattern:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        description: Schema for the entity store initialization
+        required: true
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  succeeded:
+                    type: boolean
+          description: Successful response
+      summary: Initialize the Entity Store
+      tags:
+        - Security Entity Analytics API
+      x-beta: true
   /api/entity_store/engines:
     get:
       operationId: ListEntityEngines
@@ -7713,6 +7750,27 @@ paths:
       tags:
         - Security Entity Analytics API
       x-beta: true
+  /api/entity_store/status:
+    get:
+      operationId: GetEntityStoreStatus
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  status:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_StoreStatus'
+          description: Successful response
+      summary: Get the status of the Entity Store
+      tags:
+        - Security Entity Analytics API
+      x-beta: true
   /api/exception_lists:
     delete:
       description: Delete an exception list using the `id` or `list_id` field.
@@ -45880,6 +45938,14 @@ components:
         - index
         - description
         - category
+    Security_Entity_Analytics_API_StoreStatus:
+      enum:
+        - not_installed
+        - installing
+        - running
+        - stopped
+        - error
+      type: string
     Security_Entity_Analytics_API_TaskManagerUnavailableResponse:
       description: Task manager is unavailable
       type: object

oas_docs/output/kibana.yaml

@@ -10282,6 +10282,42 @@ paths:
       summary: Create or update a protection updates note
       tags:
         - Security Endpoint Management API
+  /api/entity_store/enable:
+    post:
+      operationId: InitEntityStore
+      requestBody:
+        content:
+          application/json; Elastic-Api-Version=2023-10-31:
+            schema:
+              type: object
+              properties:
+                fieldHistoryLength:
+                  default: 10
+                  description: The number of historical values to keep for each field.
+                  type: integer
+                filter:
+                  type: string
+                indexPattern:
+                  $ref: '#/components/schemas/Security_Entity_Analytics_API_IndexPattern'
+        description: Schema for the entity store initialization
+        required: true
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  succeeded:
+                    type: boolean
+          description: Successful response
+      summary: Initialize the Entity Store
+      tags:
+        - Security Entity Analytics API
   /api/entity_store/engines:
     get:
       operationId: ListEntityEngines
@@ -10591,6 +10627,26 @@ paths:
       summary: List Entity Store Entities
       tags:
         - Security Entity Analytics API
+  /api/entity_store/status:
+    get:
+      operationId: GetEntityStoreStatus
+      responses:
+        '200':
+          content:
+            application/json; Elastic-Api-Version=2023-10-31:
+              schema:
+                type: object
+                properties:
+                  engines:
+                    items:
+                      $ref: '#/components/schemas/Security_Entity_Analytics_API_EngineDescriptor'
+                    type: array
+                  status:
+                    $ref: '#/components/schemas/Security_Entity_Analytics_API_StoreStatus'
+          description: Successful response
+      summary: Get the status of the Entity Store
+      tags:
+        - Security Entity Analytics API
   /api/exception_lists:
     delete:
       description: Delete an exception list using the `id` or `list_id` field.
@@ -53601,6 +53657,14 @@ components:
         - index
         - description
         - category
+    Security_Entity_Analytics_API_StoreStatus:
+      enum:
+        - not_installed
+        - installing
+        - running
+        - stopped
+        - error
+      type: string
     Security_Entity_Analytics_API_TaskManagerUnavailableResponse:
       description: Task manager is unavailable
       type: object

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -39,6 +39,11 @@ export const EngineDescriptor = z.object({
   error: z.object({}).optional(),
 });
 
+export type StoreStatus = z.infer<typeof StoreStatus>;
+export const StoreStatus = z.enum(['not_installed', 'installing', 'running', 'stopped', 'error']);
+export type StoreStatusEnum = typeof StoreStatus.enum;
+export const StoreStatusEnum = StoreStatus.enum;
+
 export type InspectQuery = z.infer<typeof InspectQuery>;
 export const InspectQuery = z.object({
   response: z.array(z.string()),

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -41,6 +41,15 @@ components:
         - stopped
         - updating
         - error
+
+    StoreStatus:
+      type: string
+      enum:
+        - not_installed
+        - installing
+        - running
+        - stopped
+        - error
 
     IndexPattern:
       type: string

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/enablement.gen.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * NOTICE: Do not edit this file manually.
+ * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Enable Entity Store
+ *   version: 2023-10-31
+ */
+
+import { z } from '@kbn/zod';
+
+import { IndexPattern, EngineDescriptor, StoreStatus } from './common.gen';
+
+export type GetEntityStoreStatusResponse = z.infer<typeof GetEntityStoreStatusResponse>;
+export const GetEntityStoreStatusResponse = z.object({
+  status: StoreStatus.optional(),
+  engines: z.array(EngineDescriptor).optional(),
+});
+
+export type InitEntityStoreRequestBody = z.infer<typeof InitEntityStoreRequestBody>;
+export const InitEntityStoreRequestBody = z.object({
+  /**
+   * The number of historical values to keep for each field.
+   */
+  fieldHistoryLength: z.number().int().optional().default(10),
+  indexPattern: IndexPattern.optional(),
+  filter: z.string().optional(),
+});
+export type InitEntityStoreRequestBodyInput = z.input<typeof InitEntityStoreRequestBody>;
+
+export type InitEntityStoreResponse = z.infer<typeof InitEntityStoreResponse>;
+export const InitEntityStoreResponse = z.object({
+  succeeded: z.boolean().optional(),
+  engines: z.array(EngineDescriptor).optional(),
+});

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/enablement.schema.yaml

@@ -0,0 +1,64 @@
+openapi: 3.0.0
+
+info:
+  title: Enable Entity Store
+  version: '2023-10-31'
+paths:
+  /api/entity_store/enable:
+    post:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: InitEntityStore
+      summary: Initialize the Entity Store
+
+      requestBody:
+        description: Schema for the entity store initialization
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                fieldHistoryLength:
+                  type: integer
+                  description: The number of historical values to keep for each field.
+                  default: 10
+                indexPattern:
+                  $ref: './common.schema.yaml#/components/schemas/IndexPattern'
+                filter:
+                  type: string
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  succeeded:
+                    type: boolean
+                  engines:
+                    type: array
+                    items:
+                      $ref: './common.schema.yaml#/components/schemas/EngineDescriptor'
+
+  /api/entity_store/status:
+    get:
+      x-labels: [ess, serverless]
+      x-codegen-enabled: true
+      operationId: GetEntityStoreStatus
+      summary: Get the status of the Entity Store
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status: 
+                    $ref: './common.schema.yaml#/components/schemas/StoreStatus'
+                  engines:
+                    type: array
+                    items:
+                      $ref: './common.schema.yaml#/components/schemas/EngineDescriptor'

x-pack/plugins/security_solution/common/api/quickstart_client.gen.ts

@@ -231,6 +231,11 @@ import type {
   InternalUploadAssetCriticalityRecordsResponse,
   UploadAssetCriticalityRecordsResponse,
 } from './entity_analytics/asset_criticality/upload_asset_criticality_csv.gen';
+import type {
+  GetEntityStoreStatusResponse,
+  InitEntityStoreRequestBodyInput,
+  InitEntityStoreResponse,
+} from './entity_analytics/entity_store/enablement.gen';
 import type { ApplyEntityEngineDataviewIndicesResponse } from './entity_analytics/entity_store/engine/apply_dataview_indices.gen';
 import type {
   DeleteEntityEngineRequestQueryInput,
@@ -1301,6 +1306,18 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  async getEntityStoreStatus() {
+    this.log.info(`${new Date().toISOString()} Calling API GetEntityStoreStatus`);
+    return this.kbnClient
+      .request<GetEntityStoreStatusResponse>({
+        path: '/api/entity_store/status',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
+        },
+        method: 'GET',
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Get all notes for a given document.
    */
@@ -1529,6 +1546,19 @@ finalize it.
       })
       .catch(catchAxiosErrorFormatAndThrow);
   }
+  async initEntityStore(props: InitEntityStoreProps) {
+    this.log.info(`${new Date().toISOString()} Calling API InitEntityStore`);
+    return this.kbnClient
+      .request<InitEntityStoreResponse>({
+        path: '/api/entity_store/enable',
+        headers: {
+          [ELASTIC_HTTP_VERSION_HEADER]: '2023-10-31',
+        },
+        method: 'POST',
+        body: props.body,
+      })
+      .catch(catchAxiosErrorFormatAndThrow);
+  }
   /**
    * Initializes the Risk Engine by creating the necessary indices and mappings, removing old transforms, and starting the new risk engine
    */
@@ -2290,6 +2320,9 @@ export interface InitEntityEngineProps {
   params: InitEntityEngineRequestParamsInput;
   body: InitEntityEngineRequestBodyInput;
 }
+export interface InitEntityStoreProps {
+  body: InitEntityStoreRequestBodyInput;
+}
 export interface InstallPrepackedTimelinesProps {
   body: InstallPrepackedTimelinesRequestBodyInput;
 }