From 1e980fac7a6dee3a1904511d2ae0f477575e6135 Mon Sep 17 00:00:00 2001 From: Eric Wei Date: Thu, 5 Oct 2023 19:34:07 -0700 Subject: [PATCH] [Feature] Datasource selector of multiple datasources (#5167) * initial commit for datasource Signed-off-by: Eric * initial commit for datasource service Signed-off-by: Eric * [Data Sources] Move datasource codebase from datasource plugin to data plugin Signed-off-by: Eric * [Data Sources] Add datasource factory Signed-off-by: Eric * datasource service Signed-off-by: Eric * datasource with factory Signed-off-by: Eric * datasource selector Signed-off-by: Eric * exposes datasources from data plugin Signed-off-by: Eric * index pattern datasource registration Signed-off-by: Eric * add datasource selector to discover Signed-off-by: Eric * remove unused files Signed-off-by: Eric * add/exposes types for datasources as a set of services Signed-off-by: Eric * add datasource selector unit tests Signed-off-by: Eric * add metadata to selectable Signed-off-by: Eric * redirection to observability for non-index-pattern datasource Signed-off-by: Eric * add datasource factory tests Signed-off-by: Eric * add datasource service test Signed-off-by: Eric * datasources related fixes Signed-off-by: Eric * add tests for datasource selectable Signed-off-by: Eric * added types/interfaces for sidebar selector, and add a couple of enhancements Signed-off-by: Eric * remove pill effect Signed-off-by: Eric * change type for name display in selector for index patterns Signed-off-by: Eric * remove legacy index selector Signed-off-by: Eric * rename datasource change handler Signed-off-by: Eric * remove unused constants Signed-off-by: Eric * fix a redirection bug Signed-off-by: Eric * add to change log Signed-off-by: Eric * fix bg color issue in source selector Signed-off-by: Eric * address oui and missing guard Signed-off-by: Eric * add test subj Signed-off-by: Eric * i18 and datasource interface default return Signed-off-by: Eric * add default datasource tests Signed-off-by: Eric * fix typo Signed-off-by: Eric * modify wording Signed-off-by: Eric * add experimental annotation to datasource Signed-off-by: Eric * add type datasource filtering to remove type error hint Signed-off-by: Eric * remove unused type Signed-off-by: Eric * add 'type' to option type Signed-off-by: Eric * Type fixes Signed-off-by: Ashwin P Chandran * cherry-pick type changes from remote and add partial fixes Signed-off-by: Eric * add one type exports Signed-off-by: Eric * remaining type errors fixed Signed-off-by: Ashwin P Chandran * addressing comments Signed-off-by: Eric * address dedup Signed-off-by: Eric * refactor datasource_selectable to address comments Signed-off-by: Joshua Li * remove unnecessary optional chaining Signed-off-by: Joshua Li * refactor variable names Signed-off-by: Joshua Li * move functnions out of selectable component Signed-off-by: Eric * add comments for dedup/options list updates Signed-off-by: Eric * add experimental annotation Signed-off-by: Eric * callback and experimental annotation on types Signed-off-by: Eric * handleSourceSelection callback Signed-off-by: Eric * datasource annotation and import adjustment Signed-off-by: Eric * remove pill effect Signed-off-by: Eric * minor changes addressing latest comments Signed-off-by: Eric * remove unused tests Signed-off-by: Eric --------- Signed-off-by: Eric Signed-off-by: Ashwin P Chandran Signed-off-by: Joshua Li Co-authored-by: Josh Romero Co-authored-by: Ashwin P Chandran Co-authored-by: Joshua Li --- CHANGELOG.md | 1 + .../data_sources/datasource/datasource.ts | 79 ++++++++++ .../data_sources/datasource/factory.test.ts | 93 +++++++++++ .../public/data_sources/datasource/factory.ts | 80 ++++++++++ .../public/data_sources/datasource/index.ts | 17 ++ .../public/data_sources/datasource/types.ts | 51 ++++++ .../datasource_selectable.test.tsx | 84 ++++++++++ .../datasource_selectable.tsx | 134 ++++++++++++++++ .../datasource_selector/index.tsx | 7 + .../data_sources/datasource_selector/types.ts | 33 ++++ .../datasource_service.test.ts | 118 ++++++++++++++ .../datasource_services/datasource_service.ts | 88 +++++++++++ .../data_sources/datasource_services/index.ts | 13 ++ .../data_sources/datasource_services/types.ts | 54 +++++++ .../default_datasource.test.ts | 55 +++++++ .../default_datasource/default_datasource.ts | 47 ++++++ .../data_sources/default_datasource/index.ts | 6 + .../register_default_datasource.ts | 27 ++++ src/plugins/data/public/index.ts | 26 ++++ src/plugins/data/public/plugin.ts | 13 ++ src/plugins/data/public/types.ts | 6 + .../public/components/sidebar/index.tsx | 145 ++++++++---------- src/plugins/discover/public/plugin.ts | 2 +- 23 files changed, 1100 insertions(+), 79 deletions(-) create mode 100644 src/plugins/data/public/data_sources/datasource/datasource.ts create mode 100644 src/plugins/data/public/data_sources/datasource/factory.test.ts create mode 100644 src/plugins/data/public/data_sources/datasource/factory.ts create mode 100644 src/plugins/data/public/data_sources/datasource/index.ts create mode 100644 src/plugins/data/public/data_sources/datasource/types.ts create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/index.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/types.ts create mode 100644 src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts create mode 100644 src/plugins/data/public/data_sources/datasource_services/datasource_service.ts create mode 100644 src/plugins/data/public/data_sources/datasource_services/index.ts create mode 100644 src/plugins/data/public/data_sources/datasource_services/types.ts create mode 100644 src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts create mode 100644 src/plugins/data/public/data_sources/default_datasource/default_datasource.ts create mode 100644 src/plugins/data/public/data_sources/default_datasource/index.ts create mode 100644 src/plugins/data/public/data_sources/register_default_datasource.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 936c28613952..020ce3583eba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -675,6 +675,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### 📈 Features/Enhancements +- Add DataSource service and DataSourceSelector for multiple datasource support ([#5167](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5167)) - Add extension point in saved object management to register namespaces and show filter ([#2656](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2656)) - Add updated_at column to Saved Objects' tables ([#1218](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/1218)) - Change the links in the visualize plugin to use `href` rather than `onClick` ([#2395](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2395)) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts new file mode 100644 index 000000000000..a2159c562064 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Abstract class representing a data source. This class provides foundational + * interfaces for specific data sources. Any data source connection needs to extend + * and implement from this base class + * + * DataSourceMetaData: Represents metadata associated with the data source. + * SourceDataSet: Represents the dataset associated with the data source. + * DataSourceQueryResult: Represents the result from querying the data source. + */ + +import { ConnectionStatus } from './types'; + +/** + * @experimental this class is experimental and might change in future releases. + */ +export abstract class DataSource< + DataSourceMetaData, + DataSetParams, + SourceDataSet, + DataSourceQueryParams, + DataSourceQueryResult +> { + constructor( + private readonly name: string, + private readonly type: string, + private readonly metadata: DataSourceMetaData + ) {} + + getName() { + return this.name; + } + + getType() { + return this.type; + } + + getMetadata() { + return this.metadata; + } + + /** + * Abstract method to get the dataset associated with the data source. + * Implementing classes need to provide the specific implementation. + * + * Data source selector needs to display data sources with pattern + * group (connection name) - a list of datasets. For example, get + * all available tables for flint datasources, and get all index + * patterns for OpenSearch data source + * + * @experimental This API is experimental and might change in future releases. + * @returns {SourceDataSet} Dataset associated with the data source. + */ + abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; + + /** + * Abstract method to run a query against the data source. + * Implementing classes need to provide the specific implementation. + * + * @experimental This API is experimental and might change in future releases. + * @returns {DataSourceQueryResult} Result from querying the data source. + */ + abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; + + /** + * Abstract method to test the connection to the data source. + * Implementing classes should provide the specific logic to determine + * the connection status, typically indicating success or failure. + * + * @experimental This API is experimental and might change in future releases. + * @returns {ConnectionStatus | Promise} Status of the connection test. + * @experimental + */ + abstract testConnection(): ConnectionStatus | Promise; +} diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts new file mode 100644 index 000000000000..0f9ea016748f --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/factory.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSourceFactory } from './factory'; +import { DataSource } from './datasource'; +import { IndexPattern, IndexPatternsService } from '../../index_patterns'; + +class MockDataSource extends DataSource { + private readonly indexPatterns; + + constructor({ + name, + type, + metadata, + indexPatterns, + }: { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPatternsService; + }) { + super(name, type, metadata); + this.indexPatterns = indexPatterns; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPatterns.ensureDefaultIndexPattern(); + return await this.indexPatterns.getCache(); + } + + async testConnection(): Promise { + return true; + } + + async runQuery(queryParams: any) { + return undefined; + } +} + +describe('DataSourceFactory', () => { + beforeEach(() => { + // Reset the DataSourceFactory's singleton instance before each test for isolation + (DataSourceFactory as any).factory = undefined; + }); + + it('returns a singleton instance', () => { + const instance1 = DataSourceFactory.getInstance(); + const instance2 = DataSourceFactory.getInstance(); + expect(instance1).toBe(instance2); + }); + + it('registers a new data source type correctly', () => { + const factory = DataSourceFactory.getInstance(); + expect(() => { + factory.registerDataSourceType('mock', MockDataSource); + }).not.toThrow(); + }); + + it('throws error when registering an already registered data source type', () => { + const factory = DataSourceFactory.getInstance(); + factory.registerDataSourceType('mock', MockDataSource); + expect(() => { + factory.registerDataSourceType('mock', MockDataSource); + }).toThrow('This data source type has already been registered'); + }); + + it('creates and returns an instance of the registered data source type', () => { + const factory = DataSourceFactory.getInstance(); + const mockIndexPattern = {} as IndexPattern; + const config = { + name: 'test_datasource', + type: 'mock', + metadata: null, + indexPattern: mockIndexPattern, + }; + factory.registerDataSourceType('mock', MockDataSource); + + const instance = factory.getDataSourceInstance('mock', config); + expect(instance).toBeInstanceOf(MockDataSource); + expect(instance.getName()).toEqual(config.name); + expect(instance.getType()).toEqual(config.type); + expect(instance.getMetadata()).toEqual(config.metadata); + }); + + it('throws error when trying to get an instance of an unregistered data source type', () => { + const factory = DataSourceFactory.getInstance(); + expect(() => { + factory.getDataSourceInstance('unregistered', {}); + }).toThrow('Unsupported data source type'); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts new file mode 100644 index 000000000000..f0b4e36cfb82 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -0,0 +1,80 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * The DataSourceFactory is responsible for managing the registration and creation of data source classes. + * It serves as a registry for different data source types and provides a way to instantiate them. + */ + +import { DataSourceType } from '../datasource_services'; +import { DataSource } from '../datasource'; + +type DataSourceClass< + MetaData = any, + SetParams = any, + DataSet = any, + QueryParams = any, + QueryResult = any +> = new (config: any) => DataSource; + +export class DataSourceFactory { + // Holds the singleton instance of the DataSourceFactory. + private static factory: DataSourceFactory; + + // A dictionary holding the data source type as the key and its corresponding class constructor as the value. + private dataSourceClasses: { [type: string]: DataSourceClass } = {}; + + /** + * Private constructor to ensure only one instance of DataSourceFactory is created. + */ + private constructor() {} + + /** + * Returns the singleton instance of the DataSourceFactory. If it doesn't exist, it creates one. + * + * @experimental This API is experimental and might change in future releases. + * @returns {DataSourceFactory} The single instance of DataSourceFactory. + */ + static getInstance(): DataSourceFactory { + if (!this.factory) { + this.factory = new DataSourceFactory(); + } + return this.factory; + } + + /** + * Registers a new data source type with its associated class. + * If the type has already been registered, an error is thrown. + * + * @experimental This API is experimental and might change in future releases. + * @param {string} type - The identifier for the data source type. + * @param {DataSourceClass} dataSourceClass - The constructor of the data source class. + * @throws {Error} Throws an error if the data source type has already been registered. + */ + registerDataSourceType(type: string, dataSourceClass: DataSourceClass): void { + if (this.dataSourceClasses[type]) { + throw new Error('This data source type has already been registered'); + } + this.dataSourceClasses[type] = dataSourceClass; + } + + /** + * Creates and returns an instance of the specified data source type with the given configuration. + * If the type hasn't been registered, an error is thrown. + * + * @experimental This API is experimental and might change in future releases. + * @param {string} type - The identifier for the data source type. + * @param {any} config - The configuration for the data source instance. + * @returns {DataSourceType} An instance of the specified data source type. + * @throws {Error} Throws an error if the data source type is not supported. + */ + getDataSourceInstance(type: string, config: any): DataSourceType { + const DataSourceClass = this.dataSourceClasses[type]; + if (!DataSourceClass) { + throw new Error('Unsupported data source type'); + } + return new DataSourceClass(config); + } +} diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts new file mode 100644 index 000000000000..10af40fdcfa2 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSource } from './datasource'; +export { + IDataSourceMetaData, + ISourceDataSet, + IDataSetParams, + IDataSourceQueryParams, + IDataSourceQueryResult, + ConnectionStatus, + DataSourceConfig, + IndexPatternOption, +} from './types'; +export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts new file mode 100644 index 000000000000..bf77ef123a30 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @experimental These interfaces are experimental and might change in future releases. + */ + +import { IndexPatternsService } from '../../index_patterns'; +import { DataSourceType } from '../datasource_services'; + +export interface IndexPatternOption { + title: string; + id: string; +} + +export interface IDataSourceMetaData { + name: string; +} + +export interface IDataSourceGroup { + name: string; +} + +export interface ISourceDataSet { + ds: DataSourceType; + data_sets: Array; +} + +// to-dos: add common interfaces for datasource +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface IDataSetParams {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface IDataSourceQueryParams {} + +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface IDataSourceQueryResult {} + +export interface ConnectionStatus { + success: boolean; + info: string; +} + +export interface DataSourceConfig { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPatternsService; +} diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx new file mode 100644 index 000000000000..bd424211325e --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, act } from '@testing-library/react'; +import { DataSourceSelectable } from './datasource_selectable'; +import { DataSourceType, GenericDataSource } from '../datasource_services'; +import { DataSourceGroup, DataSourceOption } from './types'; + +describe('DataSourceSelectable', () => { + let dataSourcesMock: GenericDataSource[]; + let dataSourceOptionListMock: DataSourceGroup[]; + let selectedSourcesMock: DataSourceOption[]; + let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); + let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); + let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); + + beforeEach(() => { + dataSourcesMock = [ + ({ + getDataSet: jest.fn().mockResolvedValue([]), + getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), + getName: jest.fn().mockReturnValue('SomeName'), + } as unknown) as DataSourceType, + ]; + + dataSourceOptionListMock = []; + selectedSourcesMock = []; + setSelectedSourcesMock = jest.fn(); + setDataSourceOptionListMock = jest.fn(); + onFetchDataSetErrorMock = jest.fn(); + }); + + it('renders without crashing', () => { + render( + + ); + }); + + it('fetches data sets on mount', async () => { + await act(async () => { + render( + + ); + }); + + expect(dataSourcesMock[0].getDataSet).toHaveBeenCalled(); + }); + + it('handles data set fetch errors', async () => { + (dataSourcesMock[0].getDataSet as jest.Mock).mockRejectedValue(new Error('Fetch error')); + + await act(async () => { + render( + + ); + }); + + expect(onFetchDataSetErrorMock).toHaveBeenCalledWith(new Error('Fetch error')); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx new file mode 100644 index 000000000000..77b44c206d6d --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useCallback } from 'react'; +import { EuiComboBox } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import { ISourceDataSet, IndexPatternOption } from '../datasource'; +import { DataSourceType, GenericDataSource } from '../datasource_services'; +import { DataSourceGroup, DataSourceSelectableProps } from './types'; + +type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; + +// Mapping between datasource type and its display name. +const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { + DEFAULT_INDEX_PATTERNS: 'Index patterns', + s3glue: 'Amazon S3', + spark: 'Spark', +}; + +type DataSetType = ISourceDataSet['data_sets'][number]; + +// Get data sets for a given datasource and returns it along with the source. +const getDataSetWithSource = async (ds: GenericDataSource): Promise => { + const dataSet = await ds.getDataSet(); + return { + ds, + data_sets: dataSet, + }; +}; + +// Map through all data sources and get their respective data sets. +const getDataSets = (dataSources: GenericDataSource[]) => + dataSources.map((ds) => getDataSetWithSource(ds)); + +export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { + if (typeof dataSet === 'string') return false; + + return !!(dataSet.title && dataSet.id); +}; + +// Get the option format for the combo box from the dataSource and dataSet. +export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { + const optionContent = { + type: dataSource.getType(), + name: dataSource.getName(), + ds: dataSource, + }; + if (isIndexPatterns(dataSet)) { + return { + ...optionContent, + label: dataSet.title, + value: dataSet.id, + }; + } + return { + ...optionContent, + label: dataSource.getName(), + value: dataSource.getName(), + }; +}; + +// Convert data sets into a structured format suitable for selector rendering. +const getSourceList = (allDataSets: ISourceDataSet[]) => { + const finalList = [] as DataSourceGroup[]; + allDataSets.forEach((curDataSet) => { + const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; + const groupName = DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || 'Default Group'; + + const existingGroup = finalList.find((item) => item.label === groupName); + const mappedOptions = curDataSet.data_sets.map((dataSet) => + getSourceOptions(curDataSet.ds, dataSet) + ); + + // check if add new datasource group or add to existing one + if (existingGroup) { + // options deduplication + const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); + const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + + // 'existingGroup' directly references an item in the finalList + // pushing options to 'existingGroup' updates the corresponding item in finalList + existingGroup.options.push(...nonDuplicateOptions); + } else { + finalList.push({ + label: groupName, + options: mappedOptions, + }); + } + }); + return finalList; +}; + +/** + * @experimental This component is experimental and might change in future releases. + */ +export const DataSourceSelectable = ({ + dataSources, // list of all available datasource connections. + dataSourceOptionList, // combo box renderable option list derived from dataSources + selectedSources, // current selected datasource in the form of [{ label: xxx, value: xxx }] + onDataSourceSelect, + setDataSourceOptionList, + onGetDataSetError, // onGetDataSetError, Callback for handling get data set errors. Ensure it's memoized. + singleSelection = { asPlainText: true }, +}: DataSourceSelectableProps) => { + // This effect gets data sets and prepares the datasource list for UI rendering. + useEffect(() => { + Promise.all(getDataSets(dataSources)) + .then((results) => { + setDataSourceOptionList(getSourceList(results)); + }) + .catch((e) => onGetDataSetError(e)); + }, [dataSources, setDataSourceOptionList, onGetDataSetError]); + + const handleSourceChange = useCallback( + (selectedOptions: any) => onDataSourceSelect(selectedOptions), + [onDataSourceSelect] + ); + + return ( + + ); +}; diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx new file mode 100644 index 000000000000..763c83069a6f --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSourceSelectable } from './datasource_selectable'; +export { DataSourceSelectableProps, DataSourceGroup, DataSourceOption } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts new file mode 100644 index 000000000000..87bb95ada35b --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @experimental These interfaces are experimental and might change in future releases. + */ + +import { EuiComboBoxSingleSelectionShape } from '@elastic/eui'; +import { GenericDataSource } from '../datasource_services'; + +export interface DataSourceGroup { + label: string; + options: DataSourceOption[]; +} + +export interface DataSourceOption { + label: string; + value: string; + type: string; + ds: GenericDataSource; +} + +export interface DataSourceSelectableProps { + dataSources: GenericDataSource[]; + onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; + singleSelection?: boolean | EuiComboBoxSingleSelectionShape; + onGetDataSetError: (error: Error) => void; + dataSourceOptionList: DataSourceGroup[]; + selectedSources: DataSourceOption[]; + setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; +} diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts new file mode 100644 index 000000000000..2c8f393d7093 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSource } from '../datasource'; +import { IndexPatternsService } from '../../index_patterns'; +import { DataSourceService } from '../datasource_services'; + +class MockDataSource extends DataSource { + private readonly indexPattern; + + constructor({ + name, + type, + metadata, + indexPattern, + }: { + name: string; + type: string; + metadata: any; + indexPattern: IndexPatternsService; + }) { + super(name, type, metadata); + this.indexPattern = indexPattern; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPattern.ensureDefaultIndexPattern(); + return await this.indexPattern.getCache(); + } + + async testConnection(): Promise { + return true; + } + + async runQuery(queryParams: any) { + return undefined; + } +} + +const mockIndexPattern = {} as IndexPatternsService; + +const mockConfig1 = { + name: 'test_datasource1', + type: 'mock1', + metadata: null, + indexPattern: mockIndexPattern, +}; + +const mockConfig2 = { + name: 'test_datasource2', + type: 'mock1', + metadata: null, + indexPattern: mockIndexPattern, +}; + +describe('DataSourceService', () => { + beforeEach(() => { + // Reset the DataSourceService's singleton instance before each test for isolation + (DataSourceService as any).dataSourceService = undefined; + }); + + it('returns a singleton instance', () => { + const instance1 = DataSourceService.getInstance(); + const instance2 = DataSourceService.getInstance(); + expect(instance1).toBe(instance2); + }); + + it('registers a new data source correctly', async () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + const result = await service.registerDataSource(ds); + expect(result.success).toBe(true); + }); + + it('throws error when registering an already registered data source', async () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + await service.registerDataSource(ds); + await expect(service.registerDataSource(ds)).rejects.toThrow( + 'Unable to register datasource test_datasource1, error: datasource name exists.' + ); + }); + + it('registers multiple data sources correctly', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + const results = service.registerMultipleDataSources([ds1, ds2]); + results.then((regResults) => { + expect(regResults).toHaveLength(2); + expect(regResults[0].success).toBe(true); + expect(regResults[1].success).toBe(true); + }); + }); + + it('retrieves registered data sources based on filters', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + service.registerMultipleDataSources([ds1, ds2]); + const filter = { names: ['test_datasource1'] }; + const retrievedDataSources = service.getDataSources(filter); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + }); + + it('returns all data sources if no filters provided', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + service.registerMultipleDataSources([ds1, ds2]); + const retrievedDataSources = service.getDataSources(); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).toHaveProperty('test_datasource2'); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts new file mode 100644 index 000000000000..9cf674585366 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { BehaviorSubject } from 'rxjs'; +import { + DataSourceRegistrationError, + GenericDataSource, + IDataSourceFilter, + IDataSourceRegistrationResult, +} from './types'; + +export class DataSourceService { + private static dataSourceService: DataSourceService; + // A record to store all registered data sources, using the data source name as the key. + private dataSources: Record = {}; + private dataSourcesSubject: BehaviorSubject>; + + private constructor() { + this.dataSourcesSubject = new BehaviorSubject(this.dataSources); + } + + static getInstance(): DataSourceService { + if (!this.dataSourceService) { + this.dataSourceService = new DataSourceService(); + } + return this.dataSourceService; + } + + /** + * Register multiple data sources at once. + * + * @experimental This API is experimental and might change in future releases. + * @param datasources - An array of data sources to be registered. + * @returns An array of registration results, one for each data source. + */ + async registerMultipleDataSources( + datasources: GenericDataSource[] + ): Promise { + return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); + } + + /** + * Register a single data source. + * Throws an error if a data source with the same name is already registered. + * + * @experimental This API is experimental and might change in future releases. + * @param ds - The data source to be registered. + * @returns A registration result indicating success or failure. + * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. + */ + async registerDataSource(ds: GenericDataSource): Promise { + const dsName = ds.getName(); + if (dsName in this.dataSources) { + throw new DataSourceRegistrationError( + `Unable to register datasource ${dsName}, error: datasource name exists.` + ); + } else { + this.dataSources[dsName] = ds; + this.dataSourcesSubject.next(this.dataSources); + return { success: true, info: '' } as IDataSourceRegistrationResult; + } + } + + public get dataSources$() { + return this.dataSourcesSubject.asObservable(); + } + + /** + * Retrieve the registered data sources based on provided filters. + * If no filters are provided, all registered data sources are returned. + * @experimental This API is experimental and might change in future releases. + * @param filter - An optional object with filter criteria (e.g., names of data sources). + * @returns A record of filtered data sources. + */ + getDataSources(filter?: IDataSourceFilter): Record { + if (!filter || !Array.isArray(filter.names) || filter.names.length === 0) + return this.dataSources; + + return filter.names.reduce>((filteredDataSources, dsName) => { + if (dsName in this.dataSources) { + filteredDataSources[dsName] = this.dataSources[dsName]; + } + return filteredDataSources; + }, {} as Record); + } +} diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts new file mode 100644 index 000000000000..14db278b47a5 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSourceService } from './datasource_service'; +export { + IDataSourceFilter, + IDataSourceRegistrationResult, + DataSourceRegistrationError, + DataSourceType, + GenericDataSource, +} from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts new file mode 100644 index 000000000000..cb5bb31500b4 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @experimental These interfaces, types and classes are experimental and might change + * in future releases. + */ + +import { + DataSource, + DataSourceFactory, + IDataSetParams, + IDataSourceMetaData, + IDataSourceQueryParams, + IDataSourceQueryResult, + ISourceDataSet, +} from '../datasource'; +import { DataSourceService } from './datasource_service'; + +export interface IDataSourceFilter { + names: string[]; +} + +export interface IDataSourceRegistrationResult { + success: boolean; + info: string; +} + +export class DataSourceRegistrationError extends Error { + success: boolean; + info: string; + constructor(message: string) { + super(message); + this.success = false; + this.info = message; + } +} + +export interface DataSourceStart { + dataSourceService: DataSourceService; + dataSourceFactory: DataSourceFactory; +} + +export type DataSourceType = DataSource< + IDataSourceMetaData, + IDataSetParams, + ISourceDataSet, + IDataSourceQueryParams, + IDataSourceQueryResult +>; + +export type GenericDataSource = DataSource; diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts new file mode 100644 index 000000000000..aedc6cd3853a --- /dev/null +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IndexPatternsService } from '../../index_patterns'; +import { DefaultDslDataSource } from './default_datasource'; + +describe('DefaultDslDataSource', () => { + let indexPatternsMock: IndexPatternsService; + + beforeEach(() => { + indexPatternsMock = ({ + ensureDefaultIndexPattern: jest.fn(), + getCache: jest.fn(), + } as unknown) as IndexPatternsService; + }); + + it('should ensure default index pattern and get cache', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + await dataSource.getDataSet(); + + expect(indexPatternsMock.ensureDefaultIndexPattern).toHaveBeenCalledTimes(1); + expect(indexPatternsMock.getCache).toHaveBeenCalledTimes(1); + }); + + it('should throw an error', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + await expect(dataSource.testConnection()).resolves.toBe(true); + }); + + it('should return null', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.runQuery({}); + expect(result).toBeUndefined(); + }); +}); diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts new file mode 100644 index 000000000000..c3b5d2a4cf99 --- /dev/null +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { SavedObject } from '../../../../../core/public'; +import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; +import { DataSource, DataSourceConfig, IndexPatternOption } from '../datasource'; + +export class DefaultDslDataSource extends DataSource< + any, + any, + Promise, + any, + any +> { + private readonly indexPatterns; + + constructor({ name, type, metadata, indexPatterns }: DataSourceConfig) { + super(name, type, metadata); + this.indexPatterns = indexPatterns; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPatterns.ensureDefaultIndexPattern(); + const savedObjectLst = await this.indexPatterns.getCache(); + + if (!Array.isArray(savedObjectLst)) { + return undefined; + } + + return savedObjectLst.map((savedObject: SavedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + }; + }); + } + + async testConnection(): Promise { + return true; + } + + async runQuery(queryParams: any) { + return undefined; + } +} diff --git a/src/plugins/data/public/data_sources/default_datasource/index.ts b/src/plugins/data/public/data_sources/default_datasource/index.ts new file mode 100644 index 000000000000..f612fa42f03f --- /dev/null +++ b/src/plugins/data/public/data_sources/default_datasource/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DefaultDslDataSource } from './default_datasource'; diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts new file mode 100644 index 000000000000..8dece27e82eb --- /dev/null +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { DataPublicPluginStart } from '../types'; +import { DefaultDslDataSource } from './default_datasource'; + +export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { + defaultMessage: 'OpenSearch Default', +}); + +export const registerDefaultDatasource = (data: Omit) => { + // Datasources registrations for index patterns datasource + const { dataSourceService, dataSourceFactory } = data.dataSources; + dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); + dataSourceService.registerDataSource( + dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { + name: DEFAULT_DATASOURCE_NAME, + type: DEFAULT_DATASOURCE_TYPE, + metadata: null, + indexPatterns: data.indexPatterns, + }) + ); +}; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 3f96955e22b6..3b559f9e6c63 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -496,3 +496,29 @@ export { // Export plugin after all other imports export { DataPublicPlugin as Plugin }; + +// Export datasources +export { + DataSource, + IDataSourceMetaData, + IDataSetParams, + IDataSourceQueryParams, + IDataSourceQueryResult, + ISourceDataSet, + ConnectionStatus, + DataSourceFactory, + DataSourceConfig, +} from './data_sources/datasource'; +export { + DataSourceRegistrationError, + DataSourceService, + DataSourceType, + IDataSourceFilter, + IDataSourceRegistrationResult, +} from './data_sources/datasource_services'; +export { + DataSourceSelectable, + DataSourceSelectableProps, + DataSourceGroup, + DataSourceOption, +} from './data_sources/datasource_selector'; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 5b7a262960b5..179b6c0a8c83 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -88,6 +88,9 @@ import { import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; +import { DataSourceService } from './data_sources/datasource_services'; +import { DataSourceFactory } from './data_sources/datasource'; +import { registerDefaultDatasource } from './data_sources/register_default_datasource'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -212,6 +215,10 @@ export class DataPublicPlugin uiActions.getAction(ACTION_GLOBAL_APPLY_FILTER) ); + // Create or fetch the singleton instance + const dataSourceService = DataSourceService.getInstance(); + const dataSourceFactory = DataSourceFactory.getInstance(); + const dataServices = { actions: { createFiltersFromValueClickAction, @@ -222,8 +229,14 @@ export class DataPublicPlugin indexPatterns, query, search, + dataSources: { + dataSourceService, + dataSourceFactory, + }, }; + registerDefaultDatasource(dataServices); + const SearchBar = createSearchBar({ core, data: dataServices, diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index bd1499879134..5870ea7def8e 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -41,6 +41,7 @@ import { QuerySetup, QueryStart } from './query'; import { IndexPatternsContract } from './index_patterns'; import { IndexPatternSelectProps, StatefulSearchBarProps } from './ui'; import { UsageCollectionSetup } from '../../usage_collection/public'; +import { DataSourceStart } from './data_sources/datasource_services/types'; export interface DataPublicPluginEnhancements { search: SearchEnhancements; @@ -125,6 +126,11 @@ export interface DataPublicPluginStart { * {@link DataPublicPluginStartUi} */ ui: DataPublicPluginStartUi; + /** + * multiple datasources + * {@link DataSourceStart} + */ + dataSources: DataSourceStart; } export interface IDataPluginServices extends Partial { diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 42abed642e36..14bd35656385 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,115 +3,104 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useMemo, FC, useEffect, useState } from 'react'; +import React, { FC, useCallback, useEffect, useState } from 'react'; +import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import { - EuiComboBox, - EuiSelect, - EuiComboBoxOptionOption, - EuiSpacer, - EuiSplitPanel, - EuiPageSideBar, -} from '@elastic/eui'; +import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; +import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; -import { useView } from '../../utils/use'; import { DataExplorerServices } from '../../types'; -import { useTypedDispatch, useTypedSelector, setIndexPattern } from '../../utils/state_management'; -import { setView } from '../../utils/state_management/metadata_slice'; +import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); - const [options, setOptions] = useState>>([]); - const [selectedOption, setSelectedOption] = useState>(); - const { view, viewRegistry } = useView(); - const views = viewRegistry.all(); - const viewOptions = useMemo( - () => - views.map(({ id, title }) => ({ - value: id, - text: title, - })), - [views] - ); + const [selectedSources, setSelectedSources] = useState([]); + const [dataSourceOptionList, setDataSourceOptionList] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const { services: { - data: { indexPatterns }, + data: { indexPatterns, dataSources }, notifications: { toasts }, + application, }, } = useOpenSearchDashboards(); useEffect(() => { let isMounted = true; - const fetchIndexPatterns = async () => { - await indexPatterns.ensureDefaultIndexPattern(); - const cache = await indexPatterns.getCache(); - const currentOptions = (cache || []).map((indexPattern) => ({ - label: indexPattern.attributes.title, - value: indexPattern.id, - })); - if (isMounted) { - setOptions(currentOptions); + const subscription = dataSources.dataSourceService.dataSources$.subscribe( + (currentDataSources) => { + if (isMounted) { + setActiveDataSources(Object.values(currentDataSources)); + } } - }; - fetchIndexPatterns(); + ); return () => { + subscription.unsubscribe(); isMounted = false; }; - }, [indexPatterns]); + }, [indexPatterns, dataSources]); + + const getMatchedOption = (dataSourceList: DataSourceGroup[], ipId: string) => { + for (const dsGroup of dataSourceList) { + const matchedOption = dsGroup.options.find((item) => item.value === ipId); + if (matchedOption !== undefined) return matchedOption; + } + return undefined; + }; - // Set option to the current index pattern useEffect(() => { if (indexPatternId) { - const option = options.find((o) => o.value === indexPatternId); - setSelectedOption(option); + const option = getMatchedOption(dataSourceOptionList, indexPatternId); + setSelectedSources(option ? [option] : []); } - }, [indexPatternId, options]); + }, [indexPatternId, activeDataSources, dataSourceOptionList]); + + const handleSourceSelection = useCallback( + (selectedDataSources: DataSourceOption[]) => { + if (selectedDataSources.length === 0) { + setSelectedSources(selectedDataSources); + return; + } + // Temporary redirection solution for 2.11, where clicking non-index-pattern datasource + // will redirect user to Observability event explorer + if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { + return application.navigateToUrl( + `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` + ); + } + setSelectedSources(selectedDataSources); + dispatch(setIndexPattern(selectedDataSources[0].value)); + }, + [application, dispatch] + ); + + const handleGetDataSetError = useCallback( + () => (error: Error) => { + toasts.addError(error, { + title: + i18n.translate('dataExplorer.sidebar.failedToGetDataSetErrorDescription', { + defaultMessage: 'Failed to get data set: ', + }) + (error.message || error.name), + }); + }, + [toasts] + ); return ( - { - // TODO: There are many issues with this approach, but it's a start - // 1. Combo box can delete a selected index pattern. This should not be possible - // 2. Combo box is severely truncated. This should be fixed in the EUI component - // 3. The onchange can fire with a option that is not valid. discuss where to handle this. - // 4. value is optional. If the combobox needs to act as a slecet, this should be required. - const { value } = selected[0] || {}; - - if (!value) { - toasts.addWarning({ - id: 'index-pattern-not-found', - title: i18n.translate('dataExplorer.indexPatternError', { - defaultMessage: 'Index pattern not found', - }), - }); - return; - } - - dispatch(setIndexPattern(value)); - }} + - {/* Hidden for the 2.10 release of Data Explorer. Uncomment when Data explorer is released */} - {/* - { - dispatch(setView(e.target.value)); - }} - fullWidth - /> */} {children} diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index 4d28083b8892..f8e0f254f925 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -392,7 +392,7 @@ export class DiscoverPlugin const [coreStart, deps] = await core.getStartServices(); return { executeTriggerActions: deps.uiActions.executeTriggerActions, - isEditable: () => coreStart.application.capabilities.discover.save as boolean, + isEditable: () => coreStart.application.capabilities.discover?.save as boolean, }; };