Introducing the concept of ES capabilities (#164850)

## Summary

We recently got problems because some index creation settings are
rejected by stateless ES, causing the whole system to fail and Kibana to
terminate.

We can't really use feature flags for this, given:
1. it doesn't really make sense to use manual flags for something that
strictly depend on one of our dependency's capabilities
2. we're mixing the concept of "serverless" offering and "serverless"
build. Atm we sometimes run "serverless" Kibana against traditional ES,
meaning that the "serverless" info **cannot** be used to determine if
we're connected against a default or serverless version of ES.

This was something that was agreed a few weeks back, but never acted
upon.

## Introducing ES capabilities

This PR introduces the concept of elasticsearch "capabilities".

Those capabilities are built exclusively from info coming from the ES
cluster (and not by some config flag).

This first implementation simply exposes a `serverless` flag, that is
populated depending on the `build_flavor` field of the `info` API (`/`
endpoint).

The end goal would be to expose a real capabilities (e.g "what is
supported") list instead. But ideally this would be provided by some ES
API and not by us guessing what is supported depending on the build
flavor, so for now, just exposing whether we're connected to a default
of serverless ES will suffice.

### Using it to adapt some API calls during SO migration

This PR also adapts the `createIndex` and `cloneIndex` migration action
to use this information and change their request against ES accordingly
(removing some index creation parameters that are not supported).

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/core/elasticsearch/core-elasticsearch-server-internal/index.ts

@@ -29,4 +29,5 @@ export {
 export { CoreElasticsearchRouteHandlerContext } from './src/elasticsearch_route_handler_context';
 export { retryCallCluster, migrationRetryCallCluster } from './src/retry_call_cluster';
 export { isInlineScriptingEnabled } from './src/is_scripting_enabled';
+export { getCapabilitiesFromClient } from './src/get_capabilities';
 export type { ClusterInfo } from './src/get_cluster_info';

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_service.test.mocks.ts

@@ -23,3 +23,8 @@ export const isScriptingEnabledMock = jest.fn();
 jest.doMock('./is_scripting_enabled', () => ({
   isInlineScriptingEnabled: isScriptingEnabledMock,
 }));
+
+export const getClusterInfoMock = jest.fn();
+jest.doMock('./get_cluster_info', () => ({
+  getClusterInfo$: getClusterInfoMock,
+}));

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_service.test.ts

@@ -16,10 +16,14 @@ jest.mock('./version_check/ensure_es_version', () => ({
   pollEsNodesVersion: jest.fn(),
 }));
 
-import { MockClusterClient, isScriptingEnabledMock } from './elasticsearch_service.test.mocks';
+import {
+  MockClusterClient,
+  isScriptingEnabledMock,
+  getClusterInfoMock,
+} from './elasticsearch_service.test.mocks';
 
 import type { NodesVersionCompatibility } from './version_check/ensure_es_version';
-import { BehaviorSubject, firstValueFrom } from 'rxjs';
+import { BehaviorSubject, firstValueFrom, of } from 'rxjs';
 import { first, concatMap } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/repo-info';
 import { Env } from '@kbn/config';
@@ -81,6 +85,8 @@ beforeEach(() => {
 
   isScriptingEnabledMock.mockResolvedValue(true);
 
+  getClusterInfoMock.mockReturnValue(of({}));
+
   // @ts-expect-error TS does not get that `pollEsNodesVersion` is mocked
   pollEsNodesVersionMocked.mockImplementation(pollEsNodesVersionActual);
 });
@@ -89,6 +95,7 @@ afterEach(async () => {
   jest.clearAllMocks();
   MockClusterClient.mockClear();
   isScriptingEnabledMock.mockReset();
+  getClusterInfoMock.mockReset();
   await elasticsearchService?.stop();
 });
 

packages/core/elasticsearch/core-elasticsearch-server-internal/src/elasticsearch_service.ts

@@ -21,6 +21,7 @@ import type { InternalHttpServiceSetup } from '@kbn/core-http-server-internal';
 import type {
   UnauthorizedErrorHandler,
   ElasticsearchClientConfig,
+  ElasticsearchCapabilities,
 } from '@kbn/core-elasticsearch-server';
 import { ClusterClient, AgentManager } from '@kbn/core-elasticsearch-client-server-internal';
 
@@ -37,7 +38,8 @@ import { calculateStatus$ } from './status';
 import { isValidConnection } from './is_valid_connection';
 import { isInlineScriptingEnabled } from './is_scripting_enabled';
 import { mergeConfig } from './merge_config';
-import { getClusterInfo$ } from './get_cluster_info';
+import { type ClusterInfo, getClusterInfo$ } from './get_cluster_info';
+import { getElasticsearchCapabilities } from './get_capabilities';
 
 export interface SetupDeps {
   analytics: AnalyticsServiceSetup;
@@ -57,6 +59,7 @@ export class ElasticsearchService
   private executionContextClient?: IExecutionContext;
   private esNodesCompatibility$?: Observable<NodesVersionCompatibility>;
   private client?: ClusterClient;
+  private clusterInfo$?: Observable<ClusterInfo>;
   private unauthorizedErrorHandler?: UnauthorizedErrorHandler;
   private agentManager: AgentManager;
 
@@ -104,14 +107,14 @@ export class ElasticsearchService
 
     this.esNodesCompatibility$ = esNodesCompatibility$;
 
-    const clusterInfo$ = getClusterInfo$(this.client.asInternalUser);
-    registerAnalyticsContextProvider(deps.analytics, clusterInfo$);
+    this.clusterInfo$ = getClusterInfo$(this.client.asInternalUser);
+    registerAnalyticsContextProvider(deps.analytics, this.clusterInfo$);
 
     return {
       legacy: {
         config$: this.config$,
       },
-      clusterInfo$,
+      clusterInfo$: this.clusterInfo$,
       esNodesCompatibility$,
       status$: calculateStatus$(esNodesCompatibility$),
       setUnauthorizedErrorHandler: (handler) => {
@@ -140,6 +143,8 @@ export class ElasticsearchService
       }
     });
 
+    let capabilities: ElasticsearchCapabilities;
+
     if (!config.skipStartupConnectionCheck) {
       // Ensure that the connection is established and the product is valid before moving on
       await isValidConnection(this.esNodesCompatibility$);
@@ -155,11 +160,21 @@ export class ElasticsearchService
             'Refer to https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting-security.html for more info.'
         );
       }
+
+      capabilities = getElasticsearchCapabilities({
+        clusterInfo: await firstValueFrom(this.clusterInfo$!),
+      });
+    } else {
+      // skipStartupConnectionCheck is only used for unit testing, we default to base capabilities
+      capabilities = {
+        serverless: false,
+      };
     }
 
     return {
       client: this.client!,
       createClient: (type, clientConfig) => this.createClusterClient(type, config, clientConfig),
+      getCapabilities: () => capabilities,
     };
   }
 

packages/core/elasticsearch/core-elasticsearch-server-internal/src/get_capabilities.test.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getElasticsearchCapabilities } from './get_capabilities';
+import type { ClusterInfo } from './get_cluster_info';
+
+describe('getElasticsearchCapabilities', () => {
+  const getClusterInfo = (parts: Partial<ClusterInfo>): ClusterInfo => ({
+    cluster_name: 'cluster_name',
+    cluster_uuid: 'uuid',
+    cluster_version: '13.42.9000',
+    cluster_build_flavor: 'default',
+    ...parts,
+  });
+
+  describe('capabilities.serverless', () => {
+    it('is `true` when `build_flavor` is `serverless`', () => {
+      expect(
+        getElasticsearchCapabilities({
+          clusterInfo: getClusterInfo({ cluster_build_flavor: 'serverless' }),
+        })
+      ).toEqual(
+        expect.objectContaining({
+          serverless: true,
+        })
+      );
+    });
+
+    it('is `false` when `build_flavor` is `default`', () => {
+      expect(
+        getElasticsearchCapabilities({
+          clusterInfo: getClusterInfo({ cluster_build_flavor: 'default' }),
+        })
+      ).toEqual(
+        expect.objectContaining({
+          serverless: false,
+        })
+      );
+    });
+
+    it('is `false` when `build_flavor` is a random string', () => {
+      expect(
+        getElasticsearchCapabilities({
+          clusterInfo: getClusterInfo({ cluster_build_flavor: 'some totally random string' }),
+        })
+      ).toEqual(
+        expect.objectContaining({
+          serverless: false,
+        })
+      );
+    });
+  });
+});

packages/core/elasticsearch/core-elasticsearch-server-internal/src/get_capabilities.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { firstValueFrom } from 'rxjs';
+import type {
+  ElasticsearchCapabilities,
+  ElasticsearchClient,
+} from '@kbn/core-elasticsearch-server';
+import { type ClusterInfo, getClusterInfo$ } from './get_cluster_info';
+
+const SERVERLESS_BUILD_FLAVOR = 'serverless';
+
+export const getElasticsearchCapabilities = ({
+  clusterInfo,
+}: {
+  clusterInfo: ClusterInfo;
+}): ElasticsearchCapabilities => {
+  const buildFlavor = clusterInfo.cluster_build_flavor;
+
+  return {
+    serverless: buildFlavor === SERVERLESS_BUILD_FLAVOR,
+  };
+};
+
+/**
+ * Returns the capabilities for the ES cluster the provided client is connected to.
+ *
+ * @internal
+ */
+export const getCapabilitiesFromClient = async (
+  client: ElasticsearchClient
+): Promise<ElasticsearchCapabilities> => {
+  const clusterInfo = await firstValueFrom(getClusterInfo$(client));
+  return getElasticsearchCapabilities({ clusterInfo });
+};

packages/core/elasticsearch/core-elasticsearch-server-internal/src/get_cluster_info.test.ts

@@ -21,7 +21,7 @@ describe('getClusterInfo', () => {
       number: '1.2.3',
       lucene_version: '1.2.3',
       build_date: 'DateString',
-      build_flavor: 'string',
+      build_flavor: 'default',
       build_hash: 'string',
       build_snapshot: true,
       build_type: 'string',
@@ -39,6 +39,7 @@ describe('getClusterInfo', () => {
     const context$ = getClusterInfo$(internalClient);
     await expect(firstValueFrom(context$)).resolves.toMatchInlineSnapshot(`
       Object {
+        "cluster_build_flavor": "default",
         "cluster_name": "cluster-name",
         "cluster_uuid": "cluster_uuid",
         "cluster_version": "1.2.3",
@@ -52,6 +53,7 @@ describe('getClusterInfo', () => {
     const context$ = getClusterInfo$(internalClient);
     await expect(firstValueFrom(context$)).resolves.toMatchInlineSnapshot(`
       Object {
+        "cluster_build_flavor": "default",
         "cluster_name": "cluster-name",
         "cluster_uuid": "cluster_uuid",
         "cluster_version": "1.2.3",
@@ -65,13 +67,15 @@ describe('getClusterInfo', () => {
     const context$ = getClusterInfo$(internalClient);
     await expect(firstValueFrom(context$)).resolves.toMatchInlineSnapshot(`
       Object {
+        "cluster_build_flavor": "default",
         "cluster_name": "cluster-name",
         "cluster_uuid": "cluster_uuid",
         "cluster_version": "1.2.3",
       }
     `);
     await expect(firstValueFrom(context$)).resolves.toMatchInlineSnapshot(`
       Object {
+        "cluster_build_flavor": "default",
         "cluster_name": "cluster-name",
         "cluster_uuid": "cluster_uuid",
         "cluster_version": "1.2.3",

packages/core/elasticsearch/core-elasticsearch-server-internal/src/get_cluster_info.ts

@@ -15,6 +15,7 @@ export interface ClusterInfo {
   cluster_name: string;
   cluster_uuid: string;
   cluster_version: string;
+  cluster_build_flavor: string;
 }
 
 /**
@@ -28,6 +29,7 @@ export function getClusterInfo$(internalClient: ElasticsearchClient): Observable
       cluster_name: info.cluster_name,
       cluster_uuid: info.cluster_uuid,
       cluster_version: info.version.number,
+      cluster_build_flavor: info.version.build_flavor,
     })),
     retry({ delay: 1000 }),
     shareReplay(1)

packages/core/elasticsearch/core-elasticsearch-server-internal/src/register_analytics_context_provider.test.ts

@@ -21,11 +21,17 @@ describe('registerAnalyticsContextProvider', () => {
   test('it provides the context', async () => {
     registerAnalyticsContextProvider(
       analyticsMock,
-      of({ cluster_name: 'cluster-name', cluster_uuid: 'cluster_uuid', cluster_version: '1.2.3' })
+      of({
+        cluster_name: 'cluster-name',
+        cluster_uuid: 'cluster_uuid',
+        cluster_version: '1.2.3',
+        cluster_build_flavor: 'default',
+      })
     );
     const { context$ } = analyticsMock.registerContextProvider.mock.calls[0][0];
     await expect(firstValueFrom(context$)).resolves.toMatchInlineSnapshot(`
       Object {
+        "cluster_build_flavor": "default",
         "cluster_name": "cluster-name",
         "cluster_uuid": "cluster_uuid",
         "cluster_version": "1.2.3",

packages/core/elasticsearch/core-elasticsearch-server-internal/src/register_analytics_context_provider.ts

@@ -27,6 +27,7 @@ export function registerAnalyticsContextProvider(
       cluster_name: { type: 'keyword', _meta: { description: 'The Cluster Name' } },
       cluster_uuid: { type: 'keyword', _meta: { description: 'The Cluster UUID' } },
       cluster_version: { type: 'keyword', _meta: { description: 'The Cluster version' } },
+      cluster_build_flavor: { type: 'keyword', _meta: { description: 'The Cluster build flavor' } },
     },
   });
 }

packages/core/elasticsearch/core-elasticsearch-server-mocks/src/elasticsearch_service.mock.ts

@@ -18,7 +18,9 @@ import {
 import type {
   ElasticsearchClientConfig,
   ElasticsearchServiceSetup,
+  ElasticsearchServiceStart,
   ElasticsearchServicePreboot,
+  ElasticsearchCapabilities,
 } from '@kbn/core-elasticsearch-server';
 import type {
   ElasticsearchConfig,
@@ -40,12 +42,14 @@ export type MockedElasticSearchServiceSetup = jest.Mocked<
   };
 };
 
-export interface MockedElasticSearchServiceStart {
+export type MockedElasticSearchServiceStart = jest.Mocked<
+  Omit<ElasticsearchServiceStart, 'client' | 'createClient'>
+> & {
   client: ClusterClientMock;
   createClient: jest.MockedFunction<
     (type: string, config?: Partial<ElasticsearchClientConfig>) => CustomClusterClientMock
   >;
-}
+};
 
 const createPrebootContractMock = () => {
   const prebootContract: MockedElasticSearchServicePreboot = {
@@ -69,6 +73,7 @@ const createStartContractMock = () => {
   const startContract: MockedElasticSearchServiceStart = {
     client: elasticsearchClientMock.createClusterClient(),
     createClient: jest.fn((type: string) => elasticsearchClientMock.createCustomClusterClient()),
+    getCapabilities: jest.fn().mockReturnValue(createCapabilities()),
   };
   return startContract;
 };
@@ -90,6 +95,7 @@ const createInternalSetupContractMock = () => {
       cluster_uuid: 'cluster-uuid',
       cluster_name: 'cluster-name',
       cluster_version: '8.0.0',
+      cluster_build_flavor: 'default',
     }),
     status$: new BehaviorSubject<ServiceStatus<ElasticsearchStatusMeta>>({
       level: ServiceStatusLevels.available,
@@ -117,6 +123,15 @@ const createMock = () => {
   return mocked;
 };
 
+const createCapabilities = (
+  parts: Partial<ElasticsearchCapabilities> = {}
+): ElasticsearchCapabilities => {
+  return {
+    serverless: false,
+    ...parts,
+  };
+};
+
 export const elasticsearchServiceMock = {
   create: createMock,
   createInternalPreboot: createInternalPrebootContractMock,
@@ -125,6 +140,7 @@ export const elasticsearchServiceMock = {
   createSetup: createSetupContractMock,
   createInternalStart: createInternalStartContractMock,
   createStart: createStartContractMock,
+  createCapabilities,
 
   ...elasticsearchClientMock,
 };

packages/core/elasticsearch/core-elasticsearch-server/index.ts

@@ -30,6 +30,7 @@ export type {
   ElasticsearchServicePreboot,
   ElasticsearchServiceStart,
   ElasticsearchServiceSetup,
+  ElasticsearchCapabilities,
 } from './src/contracts';
 export type { IElasticsearchConfig, ElasticsearchSslConfig } from './src/elasticsearch_config';
 export type { ElasticsearchRequestHandlerContext } from './src/request_handler_context';