[Multiple Datasource] Add installedPlugins list to data source saved … (

#6348) * [Multiple Datasource] Add installedPlugins list to data source saved object Signed-off-by: Zilong Xia <zilongx@amazon.com> * [Multiple Datasource] Fix create data source wizard test Signed-off-by: Zilong Xia <zilongx@amazon.com> --------- Signed-off-by: Zilong Xia <zilongx@amazon.com> (cherry picked from commit 7406205) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> # Conflicts: # CHANGELOG.md
opensearch-project · Apr 5, 2024 · 87d7793 · 87d7793
1 parent 88ab834
commit 87d7793
Show file tree

Hide file tree

Showing 10 changed files with 120 additions and 26 deletions.
diff --git a/src/plugins/data_source/server/plugin.ts b/src/plugins/data_source/server/plugin.ts
@@ -30,7 +30,7 @@ import { DATA_SOURCE_SAVED_OBJECT_TYPE } from '../common';
 import { ensureRawRequest } from '../../../../src/core/server/http/router';
 import { createDataSourceError } from './lib/error';
 import { registerTestConnectionRoute } from './routes/test_connection';
-import { registerFetchDataSourceVersionRoute } from './routes/fetch_data_source_version';
+import { registerFetchDataSourceMetaDataRoute } from './routes/fetch_data_source_metadata';
 import { AuthenticationMethodRegistry, IAuthenticationMethodRegistry } from './auth_registry';
 import { CustomApiSchemaRegistry } from './schema_registry';
 
@@ -134,7 +134,7 @@ export class DataSourcePlugin implements Plugin<DataSourcePluginSetup, DataSourc
       authRegistryPromise,
       customApiSchemaRegistryPromise
     );
-    registerFetchDataSourceVersionRoute(
+    registerFetchDataSourceMetaDataRoute(
       router,
       dataSourceService,
       cryptographyServiceSetup,

diff --git a/src/plugins/data_source/server/routes/data_source_connection_validator.test.ts b/src/plugins/data_source/server/routes/data_source_connection_validator.test.ts
@@ -41,6 +41,38 @@ describe('DataSourceManagement: data_source_connection_validator.ts', () => {
       expect(fetchDataSourcesVersionResponse).toBe('2.11.0');
     });
 
+    test('fetchInstalledPlugins - Success: opensearch client response code is 200 and response body have installed plugin list', async () => {
+      const opensearchClient = opensearchServiceMock.createOpenSearchClient();
+      opensearchClient.info.mockResolvedValue(
+        opensearchServiceMock.createApiResponse({
+          statusCode: 200,
+          body: [
+            {
+              name: 'b40f6833d895d3a95333e325e8bea79b',
+              component: ' analysis-icu',
+              version: '2.11.0',
+            },
+            {
+              name: 'b40f6833d895d3a95333e325e8bea79b',
+              component: 'analysis-ik',
+              version: '2.11.0',
+            },
+            {
+              name: 'b40f6833d895d3a95333e325e8bea79b',
+              component: 'analysis-seunjeon',
+              version: '2.11.0',
+            },
+          ],
+        })
+      );
+      const dataSourceValidator = new DataSourceConnectionValidator(opensearchClient, {});
+      const fetchInstalledPluginsReponse = Array.from(
+        await dataSourceValidator.fetchInstalledPlugins()
+      );
+      const installedPlugins = ['analysis-icu', 'analysis-ik', 'analysis-seunjeon'];
+      fetchInstalledPluginsReponse.map((plugin) => expect(installedPlugins).toContain(plugin));
+    });
+
     test('failure: opensearch client response code is 200 but response body not have cluster name', async () => {
       try {
         const opensearchClient = opensearchServiceMock.createOpenSearchClient();

diff --git a/src/plugins/data_source/server/routes/data_source_connection_validator.ts b/src/plugins/data_source/server/routes/data_source_connection_validator.ts
@@ -54,8 +54,37 @@ export class DataSourceConnectionValidator {
 
       return dataSourceVersion;
     } catch (e) {
-      // return empty dataSoyrce version instead of throwing exception in case info() api call fails
+      // return empty dataSource version instead of throwing exception in case info() api call fails
       return dataSourceVersion;
     }
   }
+
+  async fetchInstalledPlugins() {
+    const installedPlugins = new Set();
+    try {
+      // TODO : retrieve installed plugins from OpenSearch Serverless datasource
+      if (
+        this.dataSourceAttr.auth?.credentials?.service === SigV4ServiceName.OpenSearchServerless
+      ) {
+        return installedPlugins;
+      }
+
+      await this.callDataCluster.cat
+        .plugins({
+          format: 'JSON',
+          v: true,
+        })
+        .then((response) => response.body)
+        .then((body) => {
+          body.forEach((plugin) => {
+            installedPlugins.add(plugin.component);
+          });
+        });
+
+      return installedPlugins;
+    } catch (e) {
+      // return empty installedPlugins instead of throwing exception in case cat.plugins() api call fails
+      return installedPlugins;
+    }
+  }
 }
diff --git a/.../routes/fetch_data_source_version.test.ts → ...routes/fetch_data_source_metadata.test.ts b/.../routes/fetch_data_source_version.test.ts → ...routes/fetch_data_source_metadata.test.ts
@@ -12,16 +12,17 @@ import { authenticationMethodRegistryMock } from '../auth_registry/authenticatio
 import { CustomApiSchemaRegistry } from '../schema_registry';
 import { DataSourceServiceSetup } from '../../server/data_source_service';
 import { CryptographyServiceSetup } from '../cryptography_service';
-import { registerFetchDataSourceVersionRoute } from './fetch_data_source_version';
+import { registerFetchDataSourceMetaDataRoute } from './fetch_data_source_metadata';
 import { AuthType } from '../../common/data_sources';
 // eslint-disable-next-line @osd/eslint/no-restricted-paths
 import { opensearchClientMock } from '../../../../../src/core/server/opensearch/client/mocks';
+import { index } from 'mathjs';
 
 type SetupServerReturn = UnwrapPromise<ReturnType<typeof setupServer>>;
 
-const URL = '/internal/data-source-management/fetchDataSourceVersion';
+const URL = '/internal/data-source-management/fetchDataSourceMetaData';
 
-describe(`Fetch DataSource Version ${URL}`, () => {
+describe(`Fetch DataSource MetaData ${URL}`, () => {
   let server: SetupServerReturn['server'];
   let httpSetup: SetupServerReturn['httpSetup'];
   let handlerContext: SetupServerReturn['handlerContext'];
@@ -167,7 +168,16 @@ describe(`Fetch DataSource Version ${URL}`, () => {
     dataSourceClient.info.mockImplementationOnce(() =>
       opensearchClientMock.createSuccessTransportRequestPromise({ version: { number: '2.11.0' } })
     );
-    registerFetchDataSourceVersionRoute(
+
+    const installedPlugins = [
+      { name: 'b40f6833d895d3a95333e325e8bea79b', component: 'opensearch-ml', version: '2.11.0' },
+      { name: 'b40f6833d895d3a95333e325e8bea79b', component: 'opensearch-sql', version: '2.11.0' },
+    ];
+    dataSourceClient.cat.plugins.mockImplementationOnce(() =>
+      opensearchClientMock.createSuccessTransportRequestPromise(installedPlugins)
+    );
+
+    registerFetchDataSourceMetaDataRoute(
       router,
       dataSourceServiceSetupMock,
       cryptographyMock,
@@ -190,7 +200,10 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
     expect(dataSourceServiceSetupMock.getDataSourceClient).toHaveBeenCalledWith(
       expect.objectContaining({
         savedObjects: handlerContext.savedObjects.client,
@@ -210,7 +223,10 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr: dataSourceAttrMissingCredentialForNoAuth,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
   });
 
   it('no credential with basic auth should fail', async () => {
@@ -307,7 +323,10 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr: dataSourceAttrForSigV4Auth,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
   });
 
   it('credential with registered auth type should success', async () => {
@@ -318,7 +337,10 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr: dataSourceAttrForRegisteredAuthWithCredentials,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
   });
 
   it('empty credential with registered auth type should success', async () => {
@@ -329,7 +351,10 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr: dataSourceAttrForRegisteredAuthWithEmptyCredentials,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
   });
 
   it('no credential with registered auth type should success', async () => {
@@ -340,6 +365,9 @@ describe(`Fetch DataSource Version ${URL}`, () => {
         dataSourceAttr: dataSourceAttrForRegisteredAuthWithoutCredentials,
       })
       .expect(200);
-    expect(result.body).toEqual({ dataSourceVersion: '2.11.0' });
+    expect(result.body).toEqual({
+      dataSourceVersion: '2.11.0',
+      installedPlugins: ['opensearch-ml', 'opensearch-sql'],
+    });
   });
 });
diff --git a/...erver/routes/fetch_data_source_version.ts → ...rver/routes/fetch_data_source_metadata.ts b/...erver/routes/fetch_data_source_version.ts → ...rver/routes/fetch_data_source_metadata.ts
@@ -12,7 +12,7 @@ import { CryptographyServiceSetup } from '../cryptography_service';
 import { IAuthenticationMethodRegistry } from '../auth_registry';
 import { CustomApiSchemaRegistry } from '../schema_registry/custom_api_schema_registry';
 
-export const registerFetchDataSourceVersionRoute = async (
+export const registerFetchDataSourceMetaDataRoute = async (
   router: IRouter,
   dataSourceServiceSetup: DataSourceServiceSetup,
   cryptography: CryptographyServiceSetup,
@@ -22,7 +22,7 @@ export const registerFetchDataSourceVersionRoute = async (
   const authRegistry = await authRegistryPromise;
   router.post(
     {
-      path: '/internal/data-source-management/fetchDataSourceVersion',
+      path: '/internal/data-source-management/fetchDataSourceMetaData',
       validate: {
         body: schema.object({
           id: schema.maybe(schema.string()),
@@ -75,7 +75,6 @@ export const registerFetchDataSourceVersionRoute = async (
     },
     async (context, request, response) => {
       const { dataSourceAttr, id: dataSourceId } = request.body;
-      let dataSourceVersion = '';
 
       try {
         const dataSourceClient: OpenSearchClient = await dataSourceServiceSetup.getDataSourceClient(
@@ -95,11 +94,13 @@ export const registerFetchDataSourceVersionRoute = async (
           dataSourceAttr
         );
 
-        dataSourceVersion = await dataSourceValidator.fetchDataSourceVersion();
+        const dataSourceVersion = await dataSourceValidator.fetchDataSourceVersion();
+        const installedPlugins = Array.from(await dataSourceValidator.fetchInstalledPlugins());
 
         return response.ok({
           body: {
             dataSourceVersion,
+            installedPlugins,
           },
         });
       } catch (err) {

diff --git a/...management/public/components/create_data_source_wizard/create_data_source_wizard.test.tsx b/...management/public/components/create_data_source_wizard/create_data_source_wizard.test.tsx
@@ -5,7 +5,7 @@
 
 import React from 'react';
 import {
-  fetchDataSourceVersion,
+  fetchDataSourceMetaData,
   getMappedDataSources,
   mockDataSourceAttributesWithAuth,
   mockManagementPlugin,
@@ -28,8 +28,8 @@ describe('Datasource Management: Create Datasource Wizard', () => {
   describe('case1: should load resources successfully', () => {
     beforeEach(async () => {
       spyOn(utils, 'getDataSources').and.returnValue(Promise.resolve(getMappedDataSources));
-      spyOn(utils, 'fetchDataSourceVersion').and.returnValue(
-        Promise.resolve(fetchDataSourceVersion)
+      spyOn(utils, 'fetchDataSourceMetaData').and.returnValue(
+        Promise.resolve(fetchDataSourceMetaData)
       );
       await act(async () => {
         component = mount(

diff --git a/...urce_management/public/components/create_data_source_wizard/create_data_source_wizard.tsx b/...urce_management/public/components/create_data_source_wizard/create_data_source_wizard.tsx
@@ -20,7 +20,7 @@ import {
   createSingleDataSource,
   getDataSources,
   testConnection,
-  fetchDataSourceVersion,
+  fetchDataSourceMetaData,
   handleSetDefaultDatasource,
 } from '../utils';
 import { LoadingMask } from '../loading_mask';
@@ -75,8 +75,10 @@ export const CreateDataSourceWizard: React.FunctionComponent<CreateDataSourceWiz
   const handleSubmit = async (attributes: DataSourceAttributes) => {
     setIsLoading(true);
     try {
-      const version = await fetchDataSourceVersion(http, attributes);
-      attributes.dataSourceVersion = version.dataSourceVersion;
+      // Fetch data source metadata from added OS/ES domain/cluster
+      const metadata = await fetchDataSourceMetaData(http, attributes);
+      attributes.dataSourceVersion = metadata.dataSourceVersion;
+      attributes.installedPlugins = metadata.installedPlugins;
       await createSingleDataSource(savedObjects.client, attributes);
       // Set the first create data source as default data source.
       await handleSetDefaultDatasource(savedObjects.client, uiSettings);

diff --git a/src/plugins/data_source_management/public/components/utils.ts b/src/plugins/data_source_management/public/components/utils.ts
@@ -203,7 +203,7 @@ export async function testConnection(
   });
 }
 
-export async function fetchDataSourceVersion(
+export async function fetchDataSourceMetaData(
   http: HttpStart,
   { endpoint, auth: { type, credentials } }: DataSourceAttributes,
   dataSourceID?: string
@@ -219,7 +219,7 @@ export async function fetchDataSourceVersion(
     },
   };
 
-  return await http.post(`/internal/data-source-management/fetchDataSourceVersion`, {
+  return await http.post(`/internal/data-source-management/fetchDataSourceMetaData`, {
     body: JSON.stringify(query),
   });
 }

diff --git a/src/plugins/data_source_management/public/mocks.ts b/src/plugins/data_source_management/public/mocks.ts
@@ -232,8 +232,9 @@ export const getMappedDataSources = [
   },
 ];
 
-export const fetchDataSourceVersion = {
+export const fetchDataSourceMetaData = {
   dataSourceVersion: '2.11.0',
+  installedPlugins: ['opensearch-ml', 'opensearch-sql'],
 };
 
 export const mockDataSourceAttributesWithAuth = {

diff --git a/src/plugins/data_source_management/public/types.ts b/src/plugins/data_source_management/public/types.ts
@@ -141,6 +141,7 @@ export interface DataSourceAttributes extends SavedObjectAttributes {
   description?: string;
   endpoint?: string;
   dataSourceVersion?: string;
+  installedPlugins?: string[];
   auth: {
     type: AuthType | string;
     credentials: