From 3758626011c6f09d7b19e51482b2f78c07324a84 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 10 Aug 2023 12:55:51 +0000 Subject: [PATCH 01/56] initial commit for datasource Signed-off-by: Eric --- .../datasources/datasource/datasource.ts | 74 +++++++++++++++++++ .../public/datasources/datasource/index.ts | 13 ++++ .../public/datasources/datasource/type.ts | 16 ++++ 3 files changed, 103 insertions(+) create mode 100644 src/plugins/data_source/public/datasources/datasource/datasource.ts create mode 100644 src/plugins/data_source/public/datasources/datasource/index.ts create mode 100644 src/plugins/data_source/public/datasources/datasource/type.ts diff --git a/src/plugins/data_source/public/datasources/datasource/datasource.ts b/src/plugins/data_source/public/datasources/datasource/datasource.ts new file mode 100644 index 000000000000..19aad664f6b5 --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource/datasource.ts @@ -0,0 +1,74 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Abstract class representing a data source. This class provides foundational + * interfaces for specific data sources. Any data source connection needs to extend + * and implement from this base class + * + * DataSourceMetaData: Represents metadata associated with the data source. + * SourceDataSet: Represents the dataset associated with the data source. + * DataSourceQueryResult: Represents the result from querying the data source. + */ +export abstract class DataSource< + DataSourceMetaData, + DataSetParams, + SourceDataSet, + DataSourceQueryParams, + DataSourceQueryResult +> { + constructor( + private readonly name: string, + private readonly type: string, + private readonly metadata: DataSourceMetaData + ) {} + + getName() { + return this.name; + } + + getType() { + return this.type; + } + + getMetadata(): DataSourceMetaData { + return this.metadata; + } + + /** + * Abstract method to get the dataset associated with the data source. + * Implementing classes need to provide the specific implementation. + * + * Data source selector needs to display data sources with pattern + * group (connection name) - a list of datasets. For example, get + * all available tables for flint datasources, and get all index + * patterns for OpenSearch data source + * + * @returns {SourceDataSet} Dataset associated with the data source. + */ + abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; + + /** + * Abstract method to run a query against the data source. + * Implementing classes need to provide the specific implementation. + * + * @returns {DataSourceQueryResult} Result from querying the data source. + */ + abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; + + /** + * Abstract method to test the connection to the data source. + * Implementing classes should provide the specific logic to determine + * the connection status, typically indicating success or failure. + * + * @returns {ConnectionStatus} Status of the connection test. + */ + abstract testConnection(): ConnectionStatus; +} + +interface ConnectionStatus { + success: boolean; + info: string; +} diff --git a/src/plugins/data_source/public/datasources/datasource/index.ts b/src/plugins/data_source/public/datasources/datasource/index.ts new file mode 100644 index 000000000000..824f62e6e9b6 --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSource } from './datasource'; +export { + IDataSourceMetaData, + ISourceDataSet, + IDataSetParams, + IDataSourceQueryParams, + IDataSourceQueryResult, +} from './type'; diff --git a/src/plugins/data_source/public/datasources/datasource/type.ts b/src/plugins/data_source/public/datasources/datasource/type.ts new file mode 100644 index 000000000000..9bd37d54caed --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource/type.ts @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface IDataSourceMetaData { + name: string; +} + +export interface ISourceDataSet {} + +export interface IDataSetParams {} + +export interface IDataSourceQueryParams {} + +export interface IDataSourceQueryResult {} From 3f6b467da96a4b47ec1bd6471a22d7b8bd7d7dea Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 10 Aug 2023 13:55:57 +0000 Subject: [PATCH 02/56] initial commit for datasource service Signed-off-by: Eric --- .../datasource_services/datasource_service.ts | 86 +++++++++++++++++++ .../datasources/datasource_services/index.ts | 7 ++ .../datasources/datasource_services/types.ts | 56 ++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts create mode 100644 src/plugins/data_source/public/datasources/datasource_services/index.ts create mode 100644 src/plugins/data_source/public/datasources/datasource_services/types.ts diff --git a/src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts b/src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts new file mode 100644 index 000000000000..dca61c778376 --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { isEmpty, forEach } from 'lodash'; +import { + DataSource, + IDataSetParams, + IDataSourceMetaData, + IDataSourceQueryParams, + IDataSourceQueryResult, + ISourceDataSet, +} from '../datasource'; +import { + IDataSourceService, + IDataSourceFilters, + IDataSourceRegisterationResult, + DataSourceRegisterationError, +} from './types'; + +type DataSourceType = DataSource< + IDataSourceMetaData, + IDataSetParams, + ISourceDataSet, + IDataSourceQueryParams, + IDataSourceQueryResult +>; + +export class DataSourceService implements IDataSourceService { + // A record to store all registered data sources, using the data source name as the key. + private dataSources: Record = {}; + + constructor() {} + + /** + * Register multiple data sources at once. + * + * @param datasources - An array of data sources to be registered. + * @returns An array of registration results, one for each data source. + */ + registerMultipleDataSources(datasources: DataSourceType[]) { + return datasources.map(this.registerDatasource); + } + + /** + * Register a single data source. + * Throws an error if a data source with the same name is already registered. + * + * @param ds - The data source to be registered. + * @returns A registration result indicating success or failure. + * @throws {DataSourceRegisterationError} Throws an error if a data source with the same name already exists. + */ + async registerDatasource(ds: DataSourceType): Promise { + const dsName = ds.getName(); + if (dsName in this.dataSources) { + throw new DataSourceRegisterationError( + `Unable to register datasource ${dsName}, error: datasource name exists.` + ); + } else { + this.dataSources = { + ...this.dataSources, + [dsName]: ds, + }; + return { success: true, info: '' } as IDataSourceRegisterationResult; + } + } + + /** + * Retrieve the registered data sources based on provided filters. + * If no filters are provided, all registered data sources are returned. + * + * @param filters - An optional object with filter criteria (e.g., names of data sources). + * @returns A record of filtered data sources. + */ + getDataSources(filters?: IDataSourceFilters): Record { + if (!filters || isEmpty(filters.names)) return this.dataSources; + const filteredDataSources: Record = {}; + forEach(filters.names, (dsName) => { + if (dsName in this.dataSources) { + filteredDataSources[dsName] = this.dataSources.dsName; + } + }); + return filteredDataSources; + } +} diff --git a/src/plugins/data_source/public/datasources/datasource_services/index.ts b/src/plugins/data_source/public/datasources/datasource_services/index.ts new file mode 100644 index 000000000000..116a2dd6153a --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource_services/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSourceService } from './datasource_service'; +export { IDataSourceFilters, IDataSourceRegisterationResult, IDataSourceService } from './types'; diff --git a/src/plugins/data_source/public/datasources/datasource_services/types.ts b/src/plugins/data_source/public/datasources/datasource_services/types.ts new file mode 100644 index 000000000000..8623431ad33b --- /dev/null +++ b/src/plugins/data_source/public/datasources/datasource_services/types.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + DataSource, + IDataSetParams, + IDataSourceMetaData, + IDataSourceQueryParams, + IDataSourceQueryResult, + ISourceDataSet, +} from '../datasource'; + +export interface IDataSourceFilters { + names: string[]; +} + +export interface IDataSourceRegisterationResult { + success: boolean; + info: string; +} + +export class DataSourceRegisterationError extends Error { + success: boolean; + info: string; + constructor(message: string) { + super(message); + this.success = false; + this.info = message; + } +} + +export interface IDataSourceService { + registerDatasource: ( + ds: DataSource< + IDataSourceMetaData, + IDataSetParams, + ISourceDataSet, + IDataSourceQueryParams, + IDataSourceQueryResult + > + ) => Promise; + getDataSources: ( + filters: IDataSourceFilters + ) => Record< + string, + DataSource< + IDataSourceMetaData, + IDataSetParams, + ISourceDataSet, + IDataSourceQueryParams, + IDataSourceQueryResult + > + >; +} From 634c074f42abf5802148a6280cbb760688b4dff2 Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 19 Aug 2023 20:45:19 +0000 Subject: [PATCH 03/56] [Data Sources] Move datasource codebase from datasource plugin to data plugin Signed-off-by: Eric --- .../data_sources}/datasource/datasource.ts | 12 +- .../public/data_sources}/datasource/index.ts | 4 +- .../public/data_sources/datasource/types.ts} | 5 + .../datasource_selector/constants.ts | 15 ++ .../datasource_picker.test.tsx | 189 ++++++++++++++++++ .../datasource_selector/datasource_picker.tsx | 133 ++++++++++++ .../datasource_selector.tsx | 9 + .../datasource_services/datasource_service.ts | 16 +- .../data_sources/datasource_services/index.ts | 13 ++ .../datasource_services/types.ts | 31 +-- .../datasources/datasource_services/index.ts | 7 - 11 files changed, 394 insertions(+), 40 deletions(-) rename src/plugins/{data_source/public/datasources => data/public/data_sources}/datasource/datasource.ts (91%) rename src/plugins/{data_source/public/datasources => data/public/data_sources}/datasource/index.ts (74%) rename src/plugins/{data_source/public/datasources/datasource/type.ts => data/public/data_sources/datasource/types.ts} (80%) create mode 100644 src/plugins/data/public/data_sources/datasource_selector/constants.ts create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx rename src/plugins/{data_source/public/datasources => data/public/data_sources}/datasource_services/datasource_service.ts (85%) create mode 100644 src/plugins/data/public/data_sources/datasource_services/index.ts rename src/plugins/{data_source/public/datasources => data/public/data_sources}/datasource_services/types.ts (57%) delete mode 100644 src/plugins/data_source/public/datasources/datasource_services/index.ts diff --git a/src/plugins/data_source/public/datasources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts similarity index 91% rename from src/plugins/data_source/public/datasources/datasource/datasource.ts rename to src/plugins/data/public/data_sources/datasource/datasource.ts index 19aad664f6b5..8fe720b1cee0 100644 --- a/src/plugins/data_source/public/datasources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -12,6 +12,9 @@ * SourceDataSet: Represents the dataset associated with the data source. * DataSourceQueryResult: Represents the result from querying the data source. */ + +import { ConnectionStatus } from './types'; + export abstract class DataSource< DataSourceMetaData, DataSetParams, @@ -63,12 +66,7 @@ export abstract class DataSource< * Implementing classes should provide the specific logic to determine * the connection status, typically indicating success or failure. * - * @returns {ConnectionStatus} Status of the connection test. + * @returns {ConnectionStatus | Promise} Status of the connection test. */ - abstract testConnection(): ConnectionStatus; -} - -interface ConnectionStatus { - success: boolean; - info: string; + abstract testConnection(): ConnectionStatus | Promise; } diff --git a/src/plugins/data_source/public/datasources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts similarity index 74% rename from src/plugins/data_source/public/datasources/datasource/index.ts rename to src/plugins/data/public/data_sources/datasource/index.ts index 824f62e6e9b6..0ab60d9b8d87 100644 --- a/src/plugins/data_source/public/datasources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -10,4 +10,6 @@ export { IDataSetParams, IDataSourceQueryParams, IDataSourceQueryResult, -} from './type'; + ConnectionStatus, +} from './types'; +export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data_source/public/datasources/datasource/type.ts b/src/plugins/data/public/data_sources/datasource/types.ts similarity index 80% rename from src/plugins/data_source/public/datasources/datasource/type.ts rename to src/plugins/data/public/data_sources/datasource/types.ts index 9bd37d54caed..d9b76980d5b2 100644 --- a/src/plugins/data_source/public/datasources/datasource/type.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -14,3 +14,8 @@ export interface IDataSetParams {} export interface IDataSourceQueryParams {} export interface IDataSourceQueryResult {} + +export interface ConnectionStatus { + success: boolean; + info: string; +} diff --git a/src/plugins/data/public/data_sources/datasource_selector/constants.ts b/src/plugins/data/public/data_sources/datasource_selector/constants.ts new file mode 100644 index 000000000000..34cbf00147c7 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/constants.ts @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const SOURCE_PICKER_TITLE = 'available sources'; +export const SOURCE_PICKER_BTN_DEFAULT_TEXT = 'Select a datasource'; +export const SOURCE_PICKER_BTN_DEFAULT_WIDTH = '300px'; +export const SOURCE_PICKER_PANEL_DEFAULT_WIDTH = '500px'; +export const SOURCE_PICKER_PANEL_SEARCH_TEST_SUBJ = 'selectableDatasourcePanelSearch'; +export const SOURCE_PICKER_PANEL_TEST_SUBJ = 'selectableDatasourcePanel'; +export const SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT = 'Cancel'; +export const SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT = 'Select'; +export const SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ = 'datasourcePickerSelect'; +export const SOURCE_PICKER_BTN_TEST_SUBJ = 'sourcePickerButton'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx new file mode 100644 index 000000000000..2e7c25df4ca4 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx @@ -0,0 +1,189 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { configure, shallow, ShallowWrapper } from 'enzyme'; +import Adapter from 'enzyme-adapter-react-16'; +// @ts-ignore +import { DatasourcePicker } from './datasource_picker'; +import { IDatasourceListOption, IDatasourcePickerProps } from '../types'; +import { SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ } from './constants'; +import { OuiSelectable } from '@elastic/eui'; + +configure({ adapter: new Adapter() }); + +const dsOptions = [ + { title: '[cluster1].opensearch-dashboards-sample-data-flights' }, + { title: '[cluster1].opensearch-dashboards-sample-data-log' }, + { title: '[cluster2].opensearch-dashboards-sample-data-flights' }, + { title: '[cluster2].opensearch-dashboards-sample-data-log' }, + { title: 'opensearch-dashboards-sample-data-log' }, + { title: 'Spark_S3_sales' }, + { title: 'Spark_S3_Internal' }, +] as IDatasourceListOption[]; + +const props = { + datasourceList: [...dsOptions], + onSelect: jest.fn(), +} as IDatasourcePickerProps; + +const getSourcePickerList = (instance: ShallowWrapper) => { + return instance.find(OuiSelectable); +}; + +const getSourcePickerOptions = (instance: ShallowWrapper) => { + return getSourcePickerList(instance).prop('options'); +}; + +const selectSourcePickerOption = (instance: ShallowWrapper, selectedLabel: string) => { + const options: Array<{ label: string; checked?: 'on' | 'off' }> = getSourcePickerOptions( + instance + ).map((option: any) => + option.label === selectedLabel + ? { ...option, checked: 'on' } + : { ...option, checked: undefined } + ); + return getSourcePickerList(instance).prop('onChange')!(options); +}; + +const clickSelect = (instance: ShallowWrapper) => { + instance + .find(`[data-test-subj="${SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ}"]`) + .simulate('click'); +}; + +describe('Select datasource', () => { + it('Should allow selecting a datasource', () => { + const instance = shallow(); + selectSourcePickerOption(instance, 'Spark_S3_sales'); + clickSelect(instance); + expect(props.onSelect).toHaveBeenCalledWith({ title: 'Spark_S3_sales' }); + }); +}); + +describe('Render datasource picker', () => { + it('Should render component and match sanpshots', () => { + expect(shallow()).toMatchInlineSnapshot(` + + Select a datasource + + } + closePopover={[Function]} + display="inlineBlock" + hasArrow={true} + id="contextMenuExample" + isOpen={false} + ownFocus={true} + panelPaddingSize="none" + > + + available sources + + + + + + + + + + Cancel + + + + + Select + + + + + + `); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx new file mode 100644 index 000000000000..5d477c391e72 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx @@ -0,0 +1,133 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useMemo, useState } from 'react'; +import { + OuiPopover, + OuiButton, + OuiSelectable, + OuiPopoverFooter, + OuiFlexGroup, + OuiFlexItem, + OuiPopoverTitle, +} from '@elastic/eui'; +import { OuiSelectableLIOption } from '@elastic/eui/src/components/selectable/selectable_option'; +import { + IDatasourcePickerProps, + IDatasourceListOption, + IDatasourceSelectableOption, + ISelectedSourceOption, +} from '../types'; +import { + SOURCE_PICKER_BTN_DEFAULT_TEXT, + SOURCE_PICKER_BTN_DEFAULT_WIDTH, + SOURCE_PICKER_PANEL_DEFAULT_WIDTH, + SOURCE_PICKER_TITLE, + SOURCE_PICKER_PANEL_TEST_SUBJ, + SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT, + SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT, + SOURCE_PICKER_BTN_TEST_SUBJ, + SOURCE_PICKER_PANEL_SEARCH_TEST_SUBJ, +} from './constants'; + +export const DatasourcePicker = ({ + datasourceList, + selectedSource, + onSelect, + styles, +}: IDatasourcePickerProps) => { + const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const [selectedSourceOption, setSelectedSourceOption] = useState< + ISelectedSourceOption | undefined + >(selectedSource); + + const PickerButton = useMemo( + () => ( + setIsPopoverOpen((isOpen) => !isOpen)} + data-test-subj={SOURCE_PICKER_BTN_TEST_SUBJ} + > + {selectedSource?.title || SOURCE_PICKER_BTN_DEFAULT_TEXT} + + ), + [selectedSource, styles, setIsPopoverOpen] + ); + + const closeDatasourcePopover = () => setIsPopoverOpen(false); + + const onSelectSource = () => { + if (selectedSourceOption !== undefined) onSelect(selectedSourceOption); + closeDatasourcePopover(); + }; + + return ( + closeDatasourcePopover()} + panelPaddingSize="none" + anchorPosition="downLeft" + > + {SOURCE_PICKER_TITLE} + { + return { + label: dsItem.title, + checked: dsItem.title === selectedSource?.title ? 'on' : undefined, + } as OuiSelectableLIOption; + })} + listProps={{ bordered: true }} + onChange={(options: IDatasourceSelectableOption[]) => { + const selectedDatasource = (options.find( + ({ checked }) => checked === 'on' + ) as unknown) as IDatasourceSelectableOption; + setSelectedSourceOption({ + title: selectedDatasource.label, + }); + }} + singleSelection={true} + data-test-subj={SOURCE_PICKER_PANEL_TEST_SUBJ} + > + {(list, search) => ( + <> + {search} + {list} + + )} + + + + + + + {SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT} + + + + + {SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT} + + + + + + ); +}; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx new file mode 100644 index 000000000000..980a18616b4a --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx @@ -0,0 +1,9 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useMemo, useState } from 'react'; +import { OuiComboBox } from '@opensearch-project/oui'; + +export const DataSourceSelector = () => {}; diff --git a/src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts similarity index 85% rename from src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts rename to src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index dca61c778376..d9e232a4a3a5 100644 --- a/src/plugins/data_source/public/datasources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -19,7 +19,7 @@ import { DataSourceRegisterationError, } from './types'; -type DataSourceType = DataSource< +export type DataSourceType = DataSource< IDataSourceMetaData, IDataSetParams, ISourceDataSet, @@ -28,10 +28,18 @@ type DataSourceType = DataSource< >; export class DataSourceService implements IDataSourceService { + private static dataSourceService: IDataSourceService; // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; - constructor() {} + private constructor() {} + + static getInstance(): IDataSourceService { + if (!this.dataSourceService) { + this.dataSourceService = new DataSourceService(); + } + return this.dataSourceService; + } /** * Register multiple data sources at once. @@ -40,7 +48,7 @@ export class DataSourceService implements IDataSourceService { * @returns An array of registration results, one for each data source. */ registerMultipleDataSources(datasources: DataSourceType[]) { - return datasources.map(this.registerDatasource); + return datasources.map(this.registerDataSource); } /** @@ -51,7 +59,7 @@ export class DataSourceService implements IDataSourceService { * @returns A registration result indicating success or failure. * @throws {DataSourceRegisterationError} Throws an error if a data source with the same name already exists. */ - async registerDatasource(ds: DataSourceType): Promise { + async registerDataSource(ds: DataSourceType): Promise { const dsName = ds.getName(); if (dsName in this.dataSources) { throw new DataSourceRegisterationError( diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts new file mode 100644 index 000000000000..4bbe0dc3567b --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSourceService } from './datasource_service'; +export { + IDataSourceFilters, + IDataSourceRegisterationResult, + IDataSourceService, + DataSourceRegisterationError, + DataSourceType, +} from './types'; diff --git a/src/plugins/data_source/public/datasources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts similarity index 57% rename from src/plugins/data_source/public/datasources/datasource_services/types.ts rename to src/plugins/data/public/data_sources/datasource_services/types.ts index 8623431ad33b..4f922b13d533 100644 --- a/src/plugins/data_source/public/datasources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -31,26 +31,15 @@ export class DataSourceRegisterationError extends Error { } } +export type DataSourceType = DataSource< + IDataSourceMetaData, + IDataSetParams, + ISourceDataSet, + IDataSourceQueryParams, + IDataSourceQueryResult +>; + export interface IDataSourceService { - registerDatasource: ( - ds: DataSource< - IDataSourceMetaData, - IDataSetParams, - ISourceDataSet, - IDataSourceQueryParams, - IDataSourceQueryResult - > - ) => Promise; - getDataSources: ( - filters: IDataSourceFilters - ) => Record< - string, - DataSource< - IDataSourceMetaData, - IDataSetParams, - ISourceDataSet, - IDataSourceQueryParams, - IDataSourceQueryResult - > - >; + registerDataSource: (ds: DataSourceType) => Promise; + getDataSources: (filters?: IDataSourceFilters) => Record; } diff --git a/src/plugins/data_source/public/datasources/datasource_services/index.ts b/src/plugins/data_source/public/datasources/datasource_services/index.ts deleted file mode 100644 index 116a2dd6153a..000000000000 --- a/src/plugins/data_source/public/datasources/datasource_services/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export { DataSourceService } from './datasource_service'; -export { IDataSourceFilters, IDataSourceRegisterationResult, IDataSourceService } from './types'; From 160c73ef5de68274431102f14ea084b521cee523 Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 19 Aug 2023 20:46:54 +0000 Subject: [PATCH 04/56] [Data Sources] Add datasource factory Signed-off-by: Eric --- .../public/data_sources/datasource/factory.ts | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 src/plugins/data/public/data_sources/datasource/factory.ts diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts new file mode 100644 index 000000000000..b65013c8e840 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * The DataSourceFactory is responsible for managing the registration and creation of data source classes. + * It serves as a registry for different data source types and provides a way to instantiate them. + */ + +import { DataSourceType } from '../datasource_services'; + +export class DataSourceFactory { + // Holds the singleton instance of the DataSourceFactory. + private static factory: DataSourceFactory; + + // A dictionary holding the data source type as the key and its corresponding class constructor as the value. + private static dataSourceClasses: { [type: string]: new (config: any) => DataSourceType } = {}; + + /** + * Private constructor to ensure only one instance of DataSourceFactory is created. + */ + private constructor() {} + + /** + * Returns the singleton instance of the DataSourceFactory. If it doesn't exist, it creates one. + * + * @returns {DataSourceFactory} The single instance of DataSourceFactory. + */ + static getInstance(): DataSourceFactory { + if (!this.factory) { + this.factory = new DataSourceFactory(); + } + return this.factory; + } + + /** + * Registers a new data source type with its associated class. + * If the type has already been registered, an error is thrown. + * + * @param {string} type - The identifier for the data source type. + * @param {new (config: any) => DataSourceType} dataSourceClass - The constructor of the data source class. + * @throws {Error} Throws an error if the data source type has already been registered. + */ + static registerDataSourceType( + type: string, + dataSourceClass: new (config: any) => DataSourceType + ): void { + if (this.dataSourceClasses[type]) { + throw new Error('This data source type has already been registered'); + } + this.dataSourceClasses[type] = dataSourceClass; + } + + /** + * Creates and returns an instance of the specified data source type with the given configuration. + * If the type hasn't been registered, an error is thrown. + * + * @param {string} type - The identifier for the data source type. + * @param {any} config - The configuration for the data source instance. + * @returns {DataSourceType} An instance of the specified data source type. + * @throws {Error} Throws an error if the data source type is not supported. + */ + static getDataSourceInstance(type: string, config: any): DataSourceType { + const DataSourceClass = this.dataSourceClasses[type]; + if (!DataSourceClass) { + throw new Error('Unsupported data source type'); + } + return new DataSourceClass(config); + } +} From 12ab226cf32f52a576b1998b3add89a7ae43cc17 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 15:55:14 +0000 Subject: [PATCH 05/56] datasource service Signed-off-by: Eric --- .../datasource_services/datasource_service.ts | 18 +++++++++++++----- .../data_sources/datasource_services/index.ts | 1 - .../data_sources/datasource_services/types.ts | 5 ----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index d9e232a4a3a5..c6ccb429128e 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -3,6 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { BehaviorSubject } from 'rxjs'; import { isEmpty, forEach } from 'lodash'; import { DataSource, @@ -13,7 +14,6 @@ import { ISourceDataSet, } from '../datasource'; import { - IDataSourceService, IDataSourceFilters, IDataSourceRegisterationResult, DataSourceRegisterationError, @@ -27,14 +27,17 @@ export type DataSourceType = DataSource< IDataSourceQueryResult >; -export class DataSourceService implements IDataSourceService { - private static dataSourceService: IDataSourceService; +export class DataSourceService { + private static dataSourceService: DataSourceService; // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; + private _dataSourcesSubject: BehaviorSubject>; - private constructor() {} + private constructor() { + this._dataSourcesSubject = new BehaviorSubject(this.dataSources); + } - static getInstance(): IDataSourceService { + static getInstance(): DataSourceService { if (!this.dataSourceService) { this.dataSourceService = new DataSourceService(); } @@ -70,10 +73,15 @@ export class DataSourceService implements IDataSourceService { ...this.dataSources, [dsName]: ds, }; + this._dataSourcesSubject.next(this.dataSources); return { success: true, info: '' } as IDataSourceRegisterationResult; } } + public get dataSources$() { + return this._dataSourcesSubject.asObservable(); + } + /** * Retrieve the registered data sources based on provided filters. * If no filters are provided, all registered data sources are returned. diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts index 4bbe0dc3567b..fdd07ba9dcc9 100644 --- a/src/plugins/data/public/data_sources/datasource_services/index.ts +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -7,7 +7,6 @@ export { DataSourceService } from './datasource_service'; export { IDataSourceFilters, IDataSourceRegisterationResult, - IDataSourceService, DataSourceRegisterationError, DataSourceType, } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 4f922b13d533..85df9bf86aa6 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -38,8 +38,3 @@ export type DataSourceType = DataSource< IDataSourceQueryParams, IDataSourceQueryResult >; - -export interface IDataSourceService { - registerDataSource: (ds: DataSourceType) => Promise; - getDataSources: (filters?: IDataSourceFilters) => Record; -} From c71e0ea1c88cf81646d0851b1885d43856d8b503 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 15:56:17 +0000 Subject: [PATCH 06/56] datasource with factory Signed-off-by: Eric --- .../data/public/data_sources/datasource/factory.ts | 9 +++------ .../data/public/data_sources/datasource/types.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts index b65013c8e840..c9ed8d91e041 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -15,7 +15,7 @@ export class DataSourceFactory { private static factory: DataSourceFactory; // A dictionary holding the data source type as the key and its corresponding class constructor as the value. - private static dataSourceClasses: { [type: string]: new (config: any) => DataSourceType } = {}; + private dataSourceClasses: { [type: string]: new (config: any) => DataSourceType } = {}; /** * Private constructor to ensure only one instance of DataSourceFactory is created. @@ -42,10 +42,7 @@ export class DataSourceFactory { * @param {new (config: any) => DataSourceType} dataSourceClass - The constructor of the data source class. * @throws {Error} Throws an error if the data source type has already been registered. */ - static registerDataSourceType( - type: string, - dataSourceClass: new (config: any) => DataSourceType - ): void { + registerDataSourceType(type: string, dataSourceClass: new (config: any) => DataSourceType): void { if (this.dataSourceClasses[type]) { throw new Error('This data source type has already been registered'); } @@ -61,7 +58,7 @@ export class DataSourceFactory { * @returns {DataSourceType} An instance of the specified data source type. * @throws {Error} Throws an error if the data source type is not supported. */ - static getDataSourceInstance(type: string, config: any): DataSourceType { + getDataSourceInstance(type: string, config: any): DataSourceType { const DataSourceClass = this.dataSourceClasses[type]; if (!DataSourceClass) { throw new Error('Unsupported data source type'); diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index d9b76980d5b2..575c20227e8a 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,11 +3,21 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { IndexPatternsContract } from '../../index_patterns'; +import { DataSourceType } from '../datasource_services'; + export interface IDataSourceMetaData { name: string; } -export interface ISourceDataSet {} +export interface IDataSourceGroup { + name: string; +} + +export interface ISourceDataSet { + ds: DataSourceType; + data_sets: string[] | IndexPatternsContract; +} export interface IDataSetParams {} From ad55a8a1308e24a252b5e69f05ac288e9585a192 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 15:57:13 +0000 Subject: [PATCH 07/56] datasource selector Signed-off-by: Eric --- .../datasource_selectable.tsx | 92 +++++++++++++++++++ .../datasource_selector.tsx | 32 ++++++- .../datasource_selector/index.tsx | 7 ++ .../data_sources/datasource_selector/types.ts | 17 ++++ 4 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/index.tsx create mode 100644 src/plugins/data/public/data_sources/datasource_selector/types.ts diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx new file mode 100644 index 000000000000..7261ddc45224 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React, { useEffect, useCallback } from 'react'; +import { DataSourceSelector } from './datasource_selector'; +import { DataSourceType } from '../datasource_services'; +import { DataSourceList, DataSourceOption } from './types'; + +interface DataSourceSelectableProps { + dataSources: DataSourceType[]; + dataSourceOptionList: DataSourceList[]; + selectedSources: DataSourceOption[]; + setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; + setDataSourceOptionList: (dataSourceList: DataSourceList) => void; + singleSelection: boolean; + onFetchDataSetError: (error: Error) => void; +} + +export const DataSourceSelectable = ({ + dataSources, + dataSourceOptionList, + selectedSources, + setSelectedSources, + setDataSourceOptionList, + onFetchDataSetError, + singleSelection = true, +}: DataSourceSelectableProps) => { + const fetchDataSets = useCallback( + () => dataSources.map((ds: DataSourceType) => ds.getDataSet()), + [dataSources] + ); + + const isIndexPatterns = (dataset) => dataset.attributes; + + useEffect(() => { + const getSourceOptions = (dataSource: DataSourceType, dataSet) => { + if (isIndexPatterns(dataSet)) { + return { + label: dataSet.attributes.title, + value: dataSet.id, + ds: dataSource, + }; + } + return { label: dataSet, ds: dataSource }; + }; + + const getSourceList = (allDataSets) => { + const finalList = [] as DataSourceList[]; + allDataSets.map((curDataSet) => { + const existingGroup = finalList.find((item) => item.label === curDataSet.ds.getType()); + // check if add new datasource group or add to existing one + if (existingGroup) { + existingGroup.options = [ + ...existingGroup.options, + ...curDataSet.data_sets?.map((dataSet) => { + return getSourceOptions(curDataSet.ds, dataSet); + }), + ]; + } else { + finalList.push({ + label: curDataSet.ds.getType(), + options: curDataSet.data_sets?.map((dataSet) => { + return getSourceOptions(curDataSet.ds, dataSet); + }), + }); + } + }); + return finalList; + }; + + Promise.all(fetchDataSets()) + .then((results) => { + setDataSourceOptionList([...getSourceList(results)]); + }) + .catch((e) => onFetchDataSetError(e)); + }, [dataSources, fetchDataSets, setDataSourceOptionList, onFetchDataSetError]); + + const handleSourceChange = (selectedOptions) => { + setSelectedSources(selectedOptions); + }; + + return ( + + ); +}; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx index 980a18616b4a..993e51e125f7 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx @@ -3,7 +3,33 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useMemo, useState } from 'react'; -import { OuiComboBox } from '@opensearch-project/oui'; +import React from 'react'; +import { EuiComboBox } from '@elastic/eui'; +import { DataSourceList, DataSourceOptionType } from './types'; -export const DataSourceSelector = () => {}; +export const DataSourceSelector = ({ + dataSourceList, + selectedOptions, + onDataSourceChange, + singleSelection = true, +}: { + dataSourceList: DataSourceList[]; + selectedOptions: DataSourceOptionType[]; + onDataSourceChange: (selectedDataSourceOptions: DataSourceOptionType[]) => void; + singleSelection: boolean; +}) => { + const onDataSourceSelectionChange = (selectedDataSourceOptions: DataSourceOptionType[]) => { + onDataSourceChange(selectedDataSourceOptions); + }; + + return ( + + ); +}; diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx new file mode 100644 index 000000000000..f7a87a5ebf2d --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -0,0 +1,7 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DataSourceSelector } from './datasource_selector'; +export { DataSourceSelectable } from './datasource_selectable'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts new file mode 100644 index 000000000000..3151be1d9ad9 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { OuiComboBoxOptionOption } from '@elastic/eui'; + +export interface DataSourceList { + label: string; + options: DataSourceOption[]; +} + +export interface DataSourceOption { + label: string; +} + +export type DataSourceOptionType = OuiComboBoxOptionOption; From 5ce23631b7195ce551ca392a6dbeaca11182558a Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 16:23:56 +0000 Subject: [PATCH 08/56] exposes datasources from data plugin Signed-off-by: Eric --- src/plugins/data/public/index.ts | 21 +++++++++++++++++++++ src/plugins/data/public/plugin.ts | 12 ++++++++++++ src/plugins/data/public/types.ts | 2 ++ 3 files changed, 35 insertions(+) diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 3f96955e22b6..f1f3cced7cb6 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -496,3 +496,24 @@ export { // Export plugin after all other imports export { DataPublicPlugin as Plugin }; + +// Export datasources +export { + DataSource, + IDataSourceMetaData, + IDataSetParams, + IDataSourceQueryParams, + IDataSourceQueryResult, + ISourceDataSet, + ConnectionStatus, + DataSourceFactory, +} from './data_sources/datasource'; +export { + DataSourceRegisterationError, + DataSourceService, + DataSourceType, + IDataSourceFilters, + IDataSourceRegisterationResult, + IDataSourceService, +} from './data_sources/datasource_services'; +export { DataSourceSelector, DataSourceSelectable } from './data_sources/datasource_selector'; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 5b7a262960b5..aa61e8d36b35 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -88,6 +88,9 @@ import { import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; +import { DataSourceService } from './data_sources/datasource_services'; +import { DataSourceFactory } from './data_sources/datasource'; +import { DataSourceSelector } from './data_sources/datasource_selector/datasource_selector'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -212,6 +215,10 @@ export class DataPublicPlugin uiActions.getAction(ACTION_GLOBAL_APPLY_FILTER) ); + // Create or fetch the singleton instance + const dataSourceService = DataSourceService.getInstance(); + const dataSourceFactory = DataSourceFactory.getInstance(); + const dataServices = { actions: { createFiltersFromValueClickAction, @@ -222,6 +229,11 @@ export class DataPublicPlugin indexPatterns, query, search, + dataSources: { + dataSourceService, + dataSourceFactory, + DataSourceSelector, + }, }; const SearchBar = createSearchBar({ diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index bd1499879134..c9cdfd64e31e 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -41,6 +41,8 @@ import { QuerySetup, QueryStart } from './query'; import { IndexPatternsContract } from './index_patterns'; import { IndexPatternSelectProps, StatefulSearchBarProps } from './ui'; import { UsageCollectionSetup } from '../../usage_collection/public'; +import { IDataSourceService } from './data_sources/datasource_services'; +import { DataSourceFactory } from './data_sources/datasource'; export interface DataPublicPluginEnhancements { search: SearchEnhancements; From 004295944670c066b75d0ae731df2ed2fec9a609 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 16:58:42 +0000 Subject: [PATCH 09/56] index pattern datasource registration Signed-off-by: Eric --- src/plugins/discover/common/index.ts | 3 ++ .../public/datasources/default_datasource.ts | 46 +++++++++++++++++++ .../discover/public/datasources/index.ts | 6 +++ src/plugins/discover/public/plugin.ts | 21 ++++++++- 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 src/plugins/discover/public/datasources/default_datasource.ts create mode 100644 src/plugins/discover/public/datasources/index.ts diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index 0cac73333e25..eee1f9fcafe7 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -16,3 +16,6 @@ export const CONTEXT_DEFAULT_SIZE_SETTING = 'context:defaultSize'; export const CONTEXT_STEP_SETTING = 'context:step'; export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields'; export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch'; +export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATASOURCE_NAME = 'OpenSearch Default'; +export const INDEX_PATTERN_DATASOURCE_TYPE = 'Index patterns'; diff --git a/src/plugins/discover/public/datasources/default_datasource.ts b/src/plugins/discover/public/datasources/default_datasource.ts new file mode 100644 index 000000000000..e5fdece61453 --- /dev/null +++ b/src/plugins/discover/public/datasources/default_datasource.ts @@ -0,0 +1,46 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IndexPatternsContract } from '../../../data/public'; +import { DataSource } from '../../../data/public'; + +interface DataSourceConfig { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPatternsContract; +} + +export class DefaultDslDataSource extends DataSource< + any, + any, + Promise[] | null | undefined>, + any, + any +> { + private readonly indexPatterns; + + constructor({ name, type, metadata, indexPatterns }: DataSourceConfig) { + super(name, type, metadata); + this.indexPatterns = indexPatterns; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPatterns.ensureDefaultIndexPattern(); + const ips = await this.indexPatterns.getCache(); + return { + ds: this, // datasource instance + data_sets: ips || [], // original dataset + }; + } + + async testConnection(): Promise { + throw new Error('This operation is not supported for this class.'); + } + + async runQuery(queryParams: any) { + return null; + } +} diff --git a/src/plugins/discover/public/datasources/index.ts b/src/plugins/discover/public/datasources/index.ts new file mode 100644 index 000000000000..f612fa42f03f --- /dev/null +++ b/src/plugins/discover/public/datasources/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { DefaultDslDataSource } from './default_datasource'; diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index f1532b6f776b..527d86a9e85a 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -60,7 +60,13 @@ import { DiscoverUrlGenerator, } from './url_generator'; // import { SearchEmbeddableFactory } from './application/embeddable'; -import { NEW_DISCOVER_APP, PLUGIN_ID } from '../common'; +import { + DEFAULT_DATASOURCE_TYPE, + NEW_DISCOVER_APP, + PLUGIN_ID, + DEFAULT_DATASOURCE_NAME, + INDEX_PATTERN_DATASOURCE_TYPE, +} from '../common'; import { DataExplorerPluginSetup } from '../../data_explorer/public'; import { registerFeature } from './register_feature'; import { @@ -69,6 +75,7 @@ import { getPreloadedState, } from './application/utils/state_management/discover_slice'; import { migrateUrlState } from './migrate_state'; +import { DefaultDslDataSource } from './datasources'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -376,6 +383,18 @@ export class DiscoverPlugin this.initializeServices(); + /* Datasources registrations for default datasource (local cluster) */ + const { dataSourceService, dataSourceFactory } = plugins.data.dataSources; + dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); + dataSourceService.registerDataSource( + dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { + name: DEFAULT_DATASOURCE_NAME, + type: INDEX_PATTERN_DATASOURCE_TYPE, + metadata: null, + indexPatterns: plugins.data.indexPatterns, + }) + ); + return { urlGenerator: this.urlGenerator, savedSearchLoader: createSavedSearchesLoader({ From 1b8a7460ac77d829f60d4ebccfe54481029e86dc Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 17:01:05 +0000 Subject: [PATCH 10/56] add datasource selector to discover Signed-off-by: Eric --- .../public/components/sidebar/index.tsx | 93 ++++++++++--------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 921e3894983b..c5d5d66c57da 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,78 +3,83 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useMemo, FC, useEffect, useState } from 'react'; -import { i18n } from '@osd/i18n'; -import { - EuiComboBox, - EuiSelect, - EuiComboBoxOptionOption, - EuiSpacer, - EuiSplitPanel, - EuiPageSideBar, -} from '@elastic/eui'; +import React, { FC, useEffect, useState } from 'react'; +import { EuiSplitPanel, EuiPageSideBar } from '@elastic/eui'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; -import { useView } from '../../utils/use'; import { DataExplorerServices } from '../../types'; import { useTypedDispatch, useTypedSelector, setIndexPattern } from '../../utils/state_management'; -import { setView } from '../../utils/state_management/metadata_slice'; +import { DataSourceSelectable } from '../../../../data/public'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); - const [options, setOptions] = useState>>([]); - const [selectedOption, setSelectedOption] = useState>(); - const { view, viewRegistry } = useView(); - const views = viewRegistry.all(); - const viewOptions = useMemo( - () => - views.map(({ id, title }) => ({ - value: id, - text: title, - })), - [views] - ); + const [selectedSources, setSelectedSources] = useState([]); + const [dataSourceOptionList, setDataSourceOptionList] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const { services: { - data: { indexPatterns }, + data: { indexPatterns, dataSources }, notifications: { toasts }, + application, }, } = useOpenSearchDashboards(); useEffect(() => { let isMounted = true; - const fetchIndexPatterns = async () => { - await indexPatterns.ensureDefaultIndexPattern(); - const cache = await indexPatterns.getCache(); - const currentOptions = (cache || []).map((indexPattern) => ({ - label: indexPattern.attributes.title, - value: indexPattern.id, - })); - if (isMounted) { - setOptions(currentOptions); + const subscription = dataSources.dataSourceService.dataSources$.subscribe( + (currentDataSources) => { + if (isMounted) { + setActiveDataSources([...Object.values(currentDataSources)]); + } } - }; - fetchIndexPatterns(); + ); return () => { + subscription.unsubscribe(); isMounted = false; }; - }, [indexPatterns]); + }, [indexPatterns, dataSources]); + + const getMatchedOption = (dataSourceList, indexPatternId: string) => { + for (const dsGroup of dataSourceList) { + return dsGroup.options.find((item) => { + return item.value === indexPatternId; + }); + } + }; - // Set option to the current index pattern useEffect(() => { if (indexPatternId) { - const option = options.find((o) => o.value === indexPatternId); - setSelectedOption(option); + const option = getMatchedOption(dataSourceOptionList, indexPatternId); + setSelectedSources(option ? [option] : []); } - }, [indexPatternId, options]); + }, [indexPatternId, activeDataSources, dataSourceOptionList]); + + const handleSourceSelection = (selectedSources) => { + // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource + // will redirect user to Observability Event explorer + if (selectedSources[0].ds?.getType() !== 'Index patterns') { + return application.navigateToUrl(`../observability-logs#/explorer?metadata:($datasource:{})`); + } + setSelectedSources(selectedSources); + dispatch(setIndexPattern(selectedSources[0].value)); + }; return ( - - + + + {/* { dispatch(setIndexPattern(value)); }} - /> + /> */} {/* Hidden for the 2.10 release of Data Explorer. Uncomment when Data explorer is released */} {/* Date: Tue, 19 Sep 2023 17:07:13 +0000 Subject: [PATCH 11/56] remove unused files Signed-off-by: Eric --- .../datasource_picker.test.tsx | 189 ------------------ .../datasource_selector/datasource_picker.tsx | 133 ------------ 2 files changed, 322 deletions(-) delete mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx delete mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx deleted file mode 100644 index 2e7c25df4ca4..000000000000 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.test.tsx +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { configure, shallow, ShallowWrapper } from 'enzyme'; -import Adapter from 'enzyme-adapter-react-16'; -// @ts-ignore -import { DatasourcePicker } from './datasource_picker'; -import { IDatasourceListOption, IDatasourcePickerProps } from '../types'; -import { SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ } from './constants'; -import { OuiSelectable } from '@elastic/eui'; - -configure({ adapter: new Adapter() }); - -const dsOptions = [ - { title: '[cluster1].opensearch-dashboards-sample-data-flights' }, - { title: '[cluster1].opensearch-dashboards-sample-data-log' }, - { title: '[cluster2].opensearch-dashboards-sample-data-flights' }, - { title: '[cluster2].opensearch-dashboards-sample-data-log' }, - { title: 'opensearch-dashboards-sample-data-log' }, - { title: 'Spark_S3_sales' }, - { title: 'Spark_S3_Internal' }, -] as IDatasourceListOption[]; - -const props = { - datasourceList: [...dsOptions], - onSelect: jest.fn(), -} as IDatasourcePickerProps; - -const getSourcePickerList = (instance: ShallowWrapper) => { - return instance.find(OuiSelectable); -}; - -const getSourcePickerOptions = (instance: ShallowWrapper) => { - return getSourcePickerList(instance).prop('options'); -}; - -const selectSourcePickerOption = (instance: ShallowWrapper, selectedLabel: string) => { - const options: Array<{ label: string; checked?: 'on' | 'off' }> = getSourcePickerOptions( - instance - ).map((option: any) => - option.label === selectedLabel - ? { ...option, checked: 'on' } - : { ...option, checked: undefined } - ); - return getSourcePickerList(instance).prop('onChange')!(options); -}; - -const clickSelect = (instance: ShallowWrapper) => { - instance - .find(`[data-test-subj="${SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ}"]`) - .simulate('click'); -}; - -describe('Select datasource', () => { - it('Should allow selecting a datasource', () => { - const instance = shallow(); - selectSourcePickerOption(instance, 'Spark_S3_sales'); - clickSelect(instance); - expect(props.onSelect).toHaveBeenCalledWith({ title: 'Spark_S3_sales' }); - }); -}); - -describe('Render datasource picker', () => { - it('Should render component and match sanpshots', () => { - expect(shallow()).toMatchInlineSnapshot(` - - Select a datasource - - } - closePopover={[Function]} - display="inlineBlock" - hasArrow={true} - id="contextMenuExample" - isOpen={false} - ownFocus={true} - panelPaddingSize="none" - > - - available sources - - - - - - - - - - Cancel - - - - - Select - - - - - - `); - }); -}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx deleted file mode 100644 index 5d477c391e72..000000000000 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_picker.tsx +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React, { useMemo, useState } from 'react'; -import { - OuiPopover, - OuiButton, - OuiSelectable, - OuiPopoverFooter, - OuiFlexGroup, - OuiFlexItem, - OuiPopoverTitle, -} from '@elastic/eui'; -import { OuiSelectableLIOption } from '@elastic/eui/src/components/selectable/selectable_option'; -import { - IDatasourcePickerProps, - IDatasourceListOption, - IDatasourceSelectableOption, - ISelectedSourceOption, -} from '../types'; -import { - SOURCE_PICKER_BTN_DEFAULT_TEXT, - SOURCE_PICKER_BTN_DEFAULT_WIDTH, - SOURCE_PICKER_PANEL_DEFAULT_WIDTH, - SOURCE_PICKER_TITLE, - SOURCE_PICKER_PANEL_TEST_SUBJ, - SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT, - SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT, - SOURCE_PICKER_BTN_TEST_SUBJ, - SOURCE_PICKER_PANEL_SEARCH_TEST_SUBJ, -} from './constants'; - -export const DatasourcePicker = ({ - datasourceList, - selectedSource, - onSelect, - styles, -}: IDatasourcePickerProps) => { - const [isPopoverOpen, setIsPopoverOpen] = useState(false); - const [selectedSourceOption, setSelectedSourceOption] = useState< - ISelectedSourceOption | undefined - >(selectedSource); - - const PickerButton = useMemo( - () => ( - setIsPopoverOpen((isOpen) => !isOpen)} - data-test-subj={SOURCE_PICKER_BTN_TEST_SUBJ} - > - {selectedSource?.title || SOURCE_PICKER_BTN_DEFAULT_TEXT} - - ), - [selectedSource, styles, setIsPopoverOpen] - ); - - const closeDatasourcePopover = () => setIsPopoverOpen(false); - - const onSelectSource = () => { - if (selectedSourceOption !== undefined) onSelect(selectedSourceOption); - closeDatasourcePopover(); - }; - - return ( - closeDatasourcePopover()} - panelPaddingSize="none" - anchorPosition="downLeft" - > - {SOURCE_PICKER_TITLE} - { - return { - label: dsItem.title, - checked: dsItem.title === selectedSource?.title ? 'on' : undefined, - } as OuiSelectableLIOption; - })} - listProps={{ bordered: true }} - onChange={(options: IDatasourceSelectableOption[]) => { - const selectedDatasource = (options.find( - ({ checked }) => checked === 'on' - ) as unknown) as IDatasourceSelectableOption; - setSelectedSourceOption({ - title: selectedDatasource.label, - }); - }} - singleSelection={true} - data-test-subj={SOURCE_PICKER_PANEL_TEST_SUBJ} - > - {(list, search) => ( - <> - {search} - {list} - - )} - - - - - - - {SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT} - - - - - {SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT} - - - - - - ); -}; From 23aba4a06c1aa6f784b4055c4adbd4acffbee4a4 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 19 Sep 2023 22:32:01 +0000 Subject: [PATCH 12/56] add/exposes types for datasources as a set of services Signed-off-by: Eric --- .../datasource_selectable.tsx | 12 +---------- .../datasource_selector/index.tsx | 1 + .../data_sources/datasource_selector/types.ts | 11 ++++++++++ .../data_sources/datasource_services/types.ts | 7 +++++++ src/plugins/data/public/index.ts | 7 +++++-- src/plugins/data/public/plugin.ts | 6 ++++-- src/plugins/data/public/types.ts | 8 ++++++-- .../public/components/sidebar/index.tsx | 20 +++++++++++-------- src/plugins/data_explorer/public/types.ts | 14 ++++++++++++- src/plugins/discover/public/plugin.ts | 2 +- 10 files changed, 61 insertions(+), 27 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 7261ddc45224..861409b3bf26 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,17 +6,7 @@ import React, { useEffect, useCallback } from 'react'; import { DataSourceSelector } from './datasource_selector'; import { DataSourceType } from '../datasource_services'; -import { DataSourceList, DataSourceOption } from './types'; - -interface DataSourceSelectableProps { - dataSources: DataSourceType[]; - dataSourceOptionList: DataSourceList[]; - selectedSources: DataSourceOption[]; - setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; - setDataSourceOptionList: (dataSourceList: DataSourceList) => void; - singleSelection: boolean; - onFetchDataSetError: (error: Error) => void; -} +import { DataSourceList, DataSourceSelectableProps } from './types'; export const DataSourceSelectable = ({ dataSources, diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx index f7a87a5ebf2d..148baffd4c8b 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/index.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -5,3 +5,4 @@ export { DataSourceSelector } from './datasource_selector'; export { DataSourceSelectable } from './datasource_selectable'; +export { DataSourceSelectableProps } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 3151be1d9ad9..9bf2ced0464f 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -4,6 +4,7 @@ */ import { OuiComboBoxOptionOption } from '@elastic/eui'; +import { DataSourceType } from '../datasource_services'; export interface DataSourceList { label: string; @@ -15,3 +16,13 @@ export interface DataSourceOption { } export type DataSourceOptionType = OuiComboBoxOptionOption; + +export interface DataSourceSelectableProps { + dataSources: DataSourceType[]; + dataSourceOptionList: DataSourceList[]; + selectedSources: DataSourceOption[]; + setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; + setDataSourceOptionList: (dataSourceList: DataSourceList) => void; + singleSelection: boolean; + onFetchDataSetError: (error: Error) => void; +} diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 85df9bf86aa6..b0f4bf7d80d9 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -5,12 +5,14 @@ import { DataSource, + DataSourceFactory, IDataSetParams, IDataSourceMetaData, IDataSourceQueryParams, IDataSourceQueryResult, ISourceDataSet, } from '../datasource'; +import { DataSourceService } from './datasource_service'; export interface IDataSourceFilters { names: string[]; @@ -31,6 +33,11 @@ export class DataSourceRegisterationError extends Error { } } +export interface DataSourceStart { + dataSourceService: DataSourceService; + dataSourceFactory: DataSourceFactory; +} + export type DataSourceType = DataSource< IDataSourceMetaData, IDataSetParams, diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f1f3cced7cb6..f57803bcbeb3 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -514,6 +514,9 @@ export { DataSourceType, IDataSourceFilters, IDataSourceRegisterationResult, - IDataSourceService, } from './data_sources/datasource_services'; -export { DataSourceSelector, DataSourceSelectable } from './data_sources/datasource_selector'; +export { + DataSourceSelector, + DataSourceSelectable, + DataSourceSelectableProps, +} from './data_sources/datasource_selector'; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index aa61e8d36b35..23a29e4f8f8f 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -90,7 +90,6 @@ import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; -import { DataSourceSelector } from './data_sources/datasource_selector/datasource_selector'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -232,7 +231,6 @@ export class DataPublicPlugin dataSources: { dataSourceService, dataSourceFactory, - DataSourceSelector, }, }; @@ -248,6 +246,10 @@ export class DataPublicPlugin IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), SearchBar, }, + dataSources: { + dataSourceService, + dataSourceFactory, + }, }; } diff --git a/src/plugins/data/public/types.ts b/src/plugins/data/public/types.ts index c9cdfd64e31e..5870ea7def8e 100644 --- a/src/plugins/data/public/types.ts +++ b/src/plugins/data/public/types.ts @@ -41,8 +41,7 @@ import { QuerySetup, QueryStart } from './query'; import { IndexPatternsContract } from './index_patterns'; import { IndexPatternSelectProps, StatefulSearchBarProps } from './ui'; import { UsageCollectionSetup } from '../../usage_collection/public'; -import { IDataSourceService } from './data_sources/datasource_services'; -import { DataSourceFactory } from './data_sources/datasource'; +import { DataSourceStart } from './data_sources/datasource_services/types'; export interface DataPublicPluginEnhancements { search: SearchEnhancements; @@ -127,6 +126,11 @@ export interface DataPublicPluginStart { * {@link DataPublicPluginStartUi} */ ui: DataPublicPluginStartUi; + /** + * multiple datasources + * {@link DataSourceStart} + */ + dataSources: DataSourceStart; } export interface IDataPluginServices extends Partial { diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index c5d5d66c57da..e326e9cc5953 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -41,10 +41,10 @@ export const Sidebar: FC = ({ children }) => { }; }, [indexPatterns, dataSources]); - const getMatchedOption = (dataSourceList, indexPatternId: string) => { + const getMatchedOption = (dataSourceList, ipId: string) => { for (const dsGroup of dataSourceList) { return dsGroup.options.find((item) => { - return item.value === indexPatternId; + return item.value === ipId; }); } }; @@ -56,14 +56,18 @@ export const Sidebar: FC = ({ children }) => { } }, [indexPatternId, activeDataSources, dataSourceOptionList]); - const handleSourceSelection = (selectedSources) => { + const handleSourceSelection = (selectedDataSources) => { // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource - // will redirect user to Observability Event explorer - if (selectedSources[0].ds?.getType() !== 'Index patterns') { - return application.navigateToUrl(`../observability-logs#/explorer?metadata:($datasource:{})`); + // will redirect user to Observability event explorer + if (selectedDataSources[0].ds?.getType() !== 'Index patterns') { + return application.navigateToUrl( + `../observability-logs#/explorer?metadata:(datasource:${JSON.stringify( + selectedDataSources[0] + )})` + ); } - setSelectedSources(selectedSources); - dispatch(setIndexPattern(selectedSources[0].value)); + setSelectedSources(selectedDataSources); + dispatch(setIndexPattern(selectedDataSources[0].value)); }; return ( diff --git a/src/plugins/data_explorer/public/types.ts b/src/plugins/data_explorer/public/types.ts index 5f677fb46cfd..8bd3e51f3c3f 100644 --- a/src/plugins/data_explorer/public/types.ts +++ b/src/plugins/data_explorer/public/types.ts @@ -8,7 +8,13 @@ import { EmbeddableStart } from '../../embeddable/public'; import { ExpressionsStart } from '../../expressions/public'; import { ViewServiceStart, ViewServiceSetup } from './services/view_service'; import { IOsdUrlStateStorage } from '../../opensearch_dashboards_utils/public'; -import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public'; +import { + DataPublicPluginSetup, + DataPublicPluginStart, + DataSourceFactory, + DataSourceService, + DataSourceSelectableProps, +} from '../../data/public'; import { Store } from './utils/state_management'; export type DataExplorerPluginSetup = ViewServiceSetup; @@ -26,6 +32,12 @@ export interface DataExplorerPluginStartDependencies { data: DataPublicPluginStart; } +export interface DataSourceStart { + dataSourceService: DataSourceService; + dataSourceFactory: DataSourceFactory; + DataSourceSelector: React.ComponentType; +} + export interface DataExplorerServices extends CoreStart { store?: Store; viewRegistry: ViewServiceStart; diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index 527d86a9e85a..f06d4a3f8e59 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -383,7 +383,7 @@ export class DiscoverPlugin this.initializeServices(); - /* Datasources registrations for default datasource (local cluster) */ + // Datasources registrations for index patterns datasource const { dataSourceService, dataSourceFactory } = plugins.data.dataSources; dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); dataSourceService.registerDataSource( From 249f13954336e26d764ea59aca599ecdd4d24d7b Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 22 Sep 2023 00:54:16 +0000 Subject: [PATCH 13/56] add datasource selector unit tests Signed-off-by: Eric --- .../datasource_selector.test.tsx | 99 +++++++++++++++++++ .../datasource_selector.tsx | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx new file mode 100644 index 000000000000..a71c3b025be6 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx @@ -0,0 +1,99 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, fireEvent } from '@testing-library/react'; +import { DataSourceSelector } from './datasource_selector'; + +describe('DataSourceSelector', () => { + const mockOnDataSourceChange = jest.fn(); + + const sampleDataSources = [ + { + label: 'Index patterns', + options: [ + { label: 'sample_log1', value: 'sample_log1' }, + { label: 'sample_log2', value: 'sample_log2' }, + ], + }, + { + label: 'EMR', + options: [{ label: 'EMR_cluster', value: 'EMR_cluster' }], + }, + ]; + + const selectedSource = [{ label: 'sample_log1', value: 'sample_log1' }]; + + it('renders without crashing', () => { + const { getByText } = render( + + ); + + expect(getByText('sample_log1')).toBeInTheDocument(); + }); + + it('triggers onDataSourceChange when a data source is selected', () => { + const { getByTestId, getByText } = render( + + ); + + fireEvent.click(getByTestId('comboBoxToggleListButton')); + fireEvent.click(getByText('sample_log2')); + + expect(mockOnDataSourceChange).toHaveBeenCalledWith([ + { label: 'sample_log2', value: 'sample_log2' }, + ]); + }); + + it('has singleSelection set to true by default', () => { + const { rerender } = render( + + ); + + let comboBox = document.querySelector('[data-test-subj="comboBoxInput"]'); + expect(comboBox).toBeInTheDocument(); + + rerender( + + ); + + comboBox = document.querySelector('[data-test-subj="comboBoxInput"]'); + expect(comboBox).toBeInTheDocument(); + }); + + it('renders all data source options', () => { + const { getByText, getByTestId } = render( + + ); + + fireEvent.click(getByTestId('comboBoxToggleListButton')); + + expect(getByText('Index patterns')).toBeInTheDocument(); + expect(getByText('sample_log2')).toBeInTheDocument(); + expect(getByText('EMR')).toBeInTheDocument(); + expect(getByText('EMR_cluster')).toBeInTheDocument(); + }); +}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx index 993e51e125f7..b6501e4b054b 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx @@ -16,7 +16,7 @@ export const DataSourceSelector = ({ dataSourceList: DataSourceList[]; selectedOptions: DataSourceOptionType[]; onDataSourceChange: (selectedDataSourceOptions: DataSourceOptionType[]) => void; - singleSelection: boolean; + singleSelection?: boolean; }) => { const onDataSourceSelectionChange = (selectedDataSourceOptions: DataSourceOptionType[]) => { onDataSourceChange(selectedDataSourceOptions); From 2b8f7fff19952fd5e558bba68bf9be3223d6f573 Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 30 Sep 2023 17:16:18 +0000 Subject: [PATCH 14/56] add metadata to selectable Signed-off-by: Eric --- .../datasource_selectable.tsx | 52 ++++++++++++------- .../datasource_selector/index.tsx | 2 +- .../data_sources/datasource_selector/types.ts | 7 +-- 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 861409b3bf26..41415ba02a21 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -6,7 +6,15 @@ import React, { useEffect, useCallback } from 'react'; import { DataSourceSelector } from './datasource_selector'; import { DataSourceType } from '../datasource_services'; -import { DataSourceList, DataSourceSelectableProps } from './types'; +import { DataSourceGroup, DataSourceOptionType, DataSourceSelectableProps } from './types'; +import { ISourceDataSet } from '../datasource/types'; +import { IndexPattern } from '../../index_patterns'; + +const DATASOURCE_GRUOP_MAP = { + DEFAULT_INDEX_PATTERNS: 'Index patterns', + s3glue: 'Amazon S3', + spark: 'Spark', +}; export const DataSourceSelectable = ({ dataSources, @@ -22,38 +30,46 @@ export const DataSourceSelectable = ({ [dataSources] ); - const isIndexPatterns = (dataset) => dataset.attributes; + const isIndexPatterns = (dataset: string | IndexPattern) => + dataset.attributes?.title && dataset.id; useEffect(() => { - const getSourceOptions = (dataSource: DataSourceType, dataSet) => { + const getSourceOptions = (dataSource: DataSourceType, dataSet: string | IndexPattern) => { if (isIndexPatterns(dataSet)) { return { label: dataSet.attributes.title, value: dataSet.id, + type: dataSource.getType(), + name: dataSource.getName(), ds: dataSource, }; } - return { label: dataSet, ds: dataSource }; + return { + label: dataSet, + type: dataSource.getType(), + name: dataSource.getName(), + ds: dataSource, + }; }; - const getSourceList = (allDataSets) => { - const finalList = [] as DataSourceList[]; - allDataSets.map((curDataSet) => { - const existingGroup = finalList.find((item) => item.label === curDataSet.ds.getType()); + const getSourceList = (allDataSets: ISourceDataSet[]) => { + const finalList = [] as DataSourceGroup[]; + allDataSets.forEach((curDataSet) => { + const existingGroup = finalList.find( + (item) => item.label === DATASOURCE_GRUOP_MAP[curDataSet.ds.getType()] + ); // check if add new datasource group or add to existing one if (existingGroup) { existingGroup.options = [ ...existingGroup.options, - ...curDataSet.data_sets?.map((dataSet) => { - return getSourceOptions(curDataSet.ds, dataSet); - }), + ...curDataSet.data_sets?.map((dataSet) => getSourceOptions(curDataSet.ds, dataSet)), ]; } else { finalList.push({ - label: curDataSet.ds.getType(), - options: curDataSet.data_sets?.map((dataSet) => { - return getSourceOptions(curDataSet.ds, dataSet); - }), + label: DATASOURCE_GRUOP_MAP[curDataSet.ds.getType()] || 'Default Group', + options: curDataSet.data_sets?.map((dataSet) => + getSourceOptions(curDataSet.ds, dataSet) + ), }); } }); @@ -62,12 +78,12 @@ export const DataSourceSelectable = ({ Promise.all(fetchDataSets()) .then((results) => { - setDataSourceOptionList([...getSourceList(results)]); + setDataSourceOptionList(getSourceList(results)); }) .catch((e) => onFetchDataSetError(e)); - }, [dataSources, fetchDataSets, setDataSourceOptionList, onFetchDataSetError]); + }, [fetchDataSets, setDataSourceOptionList, onFetchDataSetError]); - const handleSourceChange = (selectedOptions) => { + const handleSourceChange = (selectedOptions: DataSourceOptionType[]) => { setSelectedSources(selectedOptions); }; diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx index 148baffd4c8b..71dc982ee0a4 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/index.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -5,4 +5,4 @@ export { DataSourceSelector } from './datasource_selector'; export { DataSourceSelectable } from './datasource_selectable'; -export { DataSourceSelectableProps } from './types'; +export { DataSourceSelectableProps, DataSourceGroup } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 9bf2ced0464f..7fc088ddac3d 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -6,23 +6,24 @@ import { OuiComboBoxOptionOption } from '@elastic/eui'; import { DataSourceType } from '../datasource_services'; -export interface DataSourceList { +export interface DataSourceGroup { label: string; options: DataSourceOption[]; } export interface DataSourceOption { label: string; + value: string; } export type DataSourceOptionType = OuiComboBoxOptionOption; export interface DataSourceSelectableProps { dataSources: DataSourceType[]; - dataSourceOptionList: DataSourceList[]; + dataSourceOptionList: DataSourceGroup[]; selectedSources: DataSourceOption[]; setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; - setDataSourceOptionList: (dataSourceList: DataSourceList) => void; + setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; singleSelection: boolean; onFetchDataSetError: (error: Error) => void; } From c8a29b711fa7938b4563287dbcbfe817c436351f Mon Sep 17 00:00:00 2001 From: Eric Date: Sat, 30 Sep 2023 22:48:16 +0000 Subject: [PATCH 15/56] redirection to observability for non-index-pattern datasource Signed-off-by: Eric --- .../data_explorer/public/components/sidebar/index.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index f080310c9b4e..3838d42eb13f 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -61,9 +61,7 @@ export const Sidebar: FC = ({ children }) => { // will redirect user to Observability event explorer if (selectedDataSources[0].ds?.getType() !== 'Index patterns') { return application.navigateToUrl( - `../observability-logs#/explorer?metadata:(datasource:${JSON.stringify( - selectedDataSources[0] - )})` + `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` ); } setSelectedSources(selectedDataSources); @@ -73,8 +71,6 @@ export const Sidebar: FC = ({ children }) => { return ( - {/* - Date: Sun, 1 Oct 2023 20:58:27 +0000 Subject: [PATCH 16/56] add datasource factory tests Signed-off-by: Eric --- .../data_sources/datasource/factory.test.ts | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/plugins/data/public/data_sources/datasource/factory.test.ts diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts new file mode 100644 index 000000000000..b80b927d9d11 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource/factory.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSourceFactory } from './factory'; +import { DataSource } from './datasource'; +import { IndexPattern } from '../../index_patterns'; + +class MockDataSource extends DataSource { + private readonly indexPatterns; + + constructor({ + name, + type, + metadata, + indexPatterns, + }: { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPattern; + }) { + super(name, type, metadata); + this.indexPatterns = indexPatterns; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPatterns.ensureDefaultIndexPattern(); + return await this.indexPatterns.getCache(); + } + + async testConnection(): Promise { + throw new Error('This operation is not supported for this class.'); + } + + async runQuery(queryParams: any) { + return null; + } +} + +describe('DataSourceFactory', () => { + beforeEach(() => { + // Reset the DataSourceFactory's singleton instance before each test for isolation + (DataSourceFactory as any).factory = undefined; + }); + + it('returns a singleton instance', () => { + const instance1 = DataSourceFactory.getInstance(); + const instance2 = DataSourceFactory.getInstance(); + expect(instance1).toBe(instance2); + }); + + it('registers a new data source type correctly', () => { + const factory = DataSourceFactory.getInstance(); + expect(() => { + factory.registerDataSourceType('mock', MockDataSource); + }).not.toThrow(); + }); + + it('throws error when registering an already registered data source type', () => { + const factory = DataSourceFactory.getInstance(); + factory.registerDataSourceType('mock', MockDataSource); + expect(() => { + factory.registerDataSourceType('mock', MockDataSource); + }).toThrow('This data source type has already been registered'); + }); + + it('creates and returns an instance of the registered data source type', () => { + const factory = DataSourceFactory.getInstance(); + const mockIndexPattern = {} as IndexPattern; + const config = { + name: 'test_datasource', + type: 'mock', + metadata: null, + indexPattern: mockIndexPattern, + }; + factory.registerDataSourceType('mock', MockDataSource); + + const instance = factory.getDataSourceInstance('mock', config); + expect(instance).toBeInstanceOf(MockDataSource); + expect(instance.getName()).toEqual(config.name); + expect(instance.getType()).toEqual(config.type); + expect(instance.getMetadata()).toEqual(config.metadata); + }); + + it('throws error when trying to get an instance of an unregistered data source type', () => { + const factory = DataSourceFactory.getInstance(); + expect(() => { + factory.getDataSourceInstance('unregistered', {}); + }).toThrow('Unsupported data source type'); + }); +}); From 4f1dc689ff0bd3128b6c1f8ce1ea93eb31527d1b Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 1 Oct 2023 22:16:15 +0000 Subject: [PATCH 17/56] add datasource service test Signed-off-by: Eric --- .../datasource_service.test.ts | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts new file mode 100644 index 000000000000..a39dfa56c12f --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSource } from '../datasource'; +import { IndexPattern } from '../../index_patterns'; +import { DataSourceService } from '../datasource_services'; + +class MockDataSource extends DataSource { + private readonly indexPatterns; + + constructor({ + name, + type, + metadata, + indexPatterns, + }: { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPattern; + }) { + super(name, type, metadata); + this.indexPatterns = indexPatterns; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPatterns.ensureDefaultIndexPattern(); + return await this.indexPatterns.getCache(); + } + + async testConnection(): Promise { + throw new Error('This operation is not supported for this class.'); + } + + async runQuery(queryParams: any) { + return null; + } +} + +const mockIndexPattern = {} as IndexPattern; + +const mockConfig1 = { + name: 'test_datasource1', + type: 'mock1', + metadata: null, + indexPattern: mockIndexPattern, +}; + +const mockConfig2 = { + name: 'test_datasource2', + type: 'mock1', + metadata: null, + indexPattern: mockIndexPattern, +}; + +describe('DataSourceService', () => { + beforeEach(() => { + // Reset the DataSourceService's singleton instance before each test for isolation + (DataSourceService as any).dataSourceService = undefined; + }); + + it('returns a singleton instance', () => { + const instance1 = DataSourceService.getInstance(); + const instance2 = DataSourceService.getInstance(); + expect(instance1).toBe(instance2); + }); + + it('registers a new data source correctly', async () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + const result = await service.registerDataSource(ds); + expect(result.success).toBe(true); + }); + + it('throws error when registering an already registered data source', async () => { + const service = DataSourceService.getInstance(); + const ds = new MockDataSource(mockConfig1); + await service.registerDataSource(ds); + await expect(service.registerDataSource(ds)).rejects.toThrow( + 'Unable to register datasource test_datasource1, error: datasource name exists.' + ); + }); + + it('registers multiple data sources correctly', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + const results = service.registerMultipleDataSources([ds1, ds2]); + results.then((regResults) => { + expect(regResults).toHaveLength(2); + expect(regResults[0].success).toBe(true); + expect(regResults[1].success).toBe(true); + }); + }); + + it('retrieves registered data sources based on filters', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + service.registerMultipleDataSources([ds1, ds2]); + const filters = { names: ['test_datasource1'] }; + const retrievedDataSources = service.getDataSources(filters); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); + }); + + it('returns all data sources if no filters provided', () => { + const service = DataSourceService.getInstance(); + const ds1 = new MockDataSource(mockConfig1); + const ds2 = new MockDataSource(mockConfig2); + service.registerMultipleDataSources([ds1, ds2]); + const retrievedDataSources = service.getDataSources(); + expect(retrievedDataSources).toHaveProperty('test_datasource1'); + expect(retrievedDataSources).toHaveProperty('test_datasource2'); + }); +}); From 6e8c2d109f98e414c33c00eeb2290a1b9e1364e8 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 00:51:45 +0000 Subject: [PATCH 18/56] datasources related fixes Signed-off-by: Eric --- .../public/data_sources/datasource/types.ts | 8 +- .../datasource_selectable.tsx | 73 ++++++++++++------- .../data_sources/datasource_selector/types.ts | 2 +- .../datasource_services/datasource_service.ts | 8 +- .../public/datasources/default_datasource.ts | 10 +-- 5 files changed, 61 insertions(+), 40 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 575c20227e8a..26e5023c2a3d 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IndexPatternsContract } from '../../index_patterns'; +import { IndexPattern } from '../../index_patterns'; import { DataSourceType } from '../datasource_services'; export interface IDataSourceMetaData { @@ -16,13 +16,17 @@ export interface IDataSourceGroup { export interface ISourceDataSet { ds: DataSourceType; - data_sets: string[] | IndexPatternsContract; + data_sets: string[] | IndexPattern; } +// to-dos: add common interfaces for datasource +// eslint-disable-next-line @typescript-eslint/no-empty-interface export interface IDataSetParams {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface export interface IDataSourceQueryParams {} +// eslint-disable-next-line @typescript-eslint/no-empty-interface export interface IDataSourceQueryResult {} export interface ConnectionStatus { diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 41415ba02a21..b1628c751ec8 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -3,71 +3,88 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect, useCallback } from 'react'; +import React, { useEffect } from 'react'; import { DataSourceSelector } from './datasource_selector'; import { DataSourceType } from '../datasource_services'; -import { DataSourceGroup, DataSourceOptionType, DataSourceSelectableProps } from './types'; +import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './types'; import { ISourceDataSet } from '../datasource/types'; import { IndexPattern } from '../../index_patterns'; -const DATASOURCE_GRUOP_MAP = { +// Mapping between datasource type and its display name. +const DATASOURCE_TYPE_DISPLAY_NAME_MAP = { DEFAULT_INDEX_PATTERNS: 'Index patterns', s3glue: 'Amazon S3', spark: 'Spark', }; +type DataSetType = string | IndexPattern; + export const DataSourceSelectable = ({ - dataSources, - dataSourceOptionList, - selectedSources, + dataSources, // list of all available datasource connections. + dataSourceOptionList, // combo box renderable option list derived from dataSources + selectedSources, // current selected datasource in the form of [{ label: xxx, value: xxx }] setSelectedSources, setDataSourceOptionList, - onFetchDataSetError, + onFetchDataSetError, // onFetchDataSetError, Callback for handling dataset fetch errors. Ensure it's memoized. singleSelection = true, }: DataSourceSelectableProps) => { - const fetchDataSets = useCallback( - () => dataSources.map((ds: DataSourceType) => ds.getDataSet()), - [dataSources] - ); + // This effect fetches datasets and prepares the datasource list for UI rendering. + useEffect(() => { + // Fetches datasets for a given datasource and returns it along with the source. + const fetchDataSetWithSource = async (ds: DataSourceType): Promise => { + const dataSet = await ds.getDataSet(); + return { + ds, + data_sets: dataSet, + }; + }; - const isIndexPatterns = (dataset: string | IndexPattern) => - dataset.attributes?.title && dataset.id; + // Map through all data sources and fetch their respective datasets. + const fetchDataSets = () => dataSources.map((ds: DataSourceType) => fetchDataSetWithSource(ds)); - useEffect(() => { - const getSourceOptions = (dataSource: DataSourceType, dataSet: string | IndexPattern) => { + const isIndexPatterns = (dataset: string | IndexPattern) => + dataset.attributes?.title && dataset.id; + + // Get the option format for the combo box from the dataSource and dataSet. + const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { + const optionContent = { + type: dataSource.getType(), + name: dataSource.getName(), + ds: dataSource, + }; if (isIndexPatterns(dataSet)) { return { + ...optionContent, label: dataSet.attributes.title, value: dataSet.id, - type: dataSource.getType(), - name: dataSource.getName(), - ds: dataSource, }; } return { - label: dataSet, - type: dataSource.getType(), - name: dataSource.getName(), - ds: dataSource, + ...optionContent, + label: dataSource.getName(), + value: dataSource.getName(), }; }; + // Convert fetched datasets into a structured format suitable for selector rendering. const getSourceList = (allDataSets: ISourceDataSet[]) => { const finalList = [] as DataSourceGroup[]; allDataSets.forEach((curDataSet) => { const existingGroup = finalList.find( - (item) => item.label === DATASOURCE_GRUOP_MAP[curDataSet.ds.getType()] + (item) => item.label === DATASOURCE_TYPE_DISPLAY_NAME_MAP[curDataSet.ds.getType()] ); // check if add new datasource group or add to existing one if (existingGroup) { existingGroup.options = [ ...existingGroup.options, - ...curDataSet.data_sets?.map((dataSet) => getSourceOptions(curDataSet.ds, dataSet)), + ...curDataSet.data_sets?.map((dataSet: DataSetType) => + getSourceOptions(curDataSet.ds, dataSet) + ), ]; } else { finalList.push({ - label: DATASOURCE_GRUOP_MAP[curDataSet.ds.getType()] || 'Default Group', - options: curDataSet.data_sets?.map((dataSet) => + label: DATASOURCE_TYPE_DISPLAY_NAME_MAP[curDataSet.ds.getType()] || 'Default Group', + options: curDataSet.data_sets?.map((dataSet: DataSetType) => getSourceOptions(curDataSet.ds, dataSet) ), }); @@ -81,9 +98,9 @@ export const DataSourceSelectable = ({ setDataSourceOptionList(getSourceList(results)); }) .catch((e) => onFetchDataSetError(e)); - }, [fetchDataSets, setDataSourceOptionList, onFetchDataSetError]); + }, [dataSources, setDataSourceOptionList, onFetchDataSetError]); - const handleSourceChange = (selectedOptions: DataSourceOptionType[]) => { + const handleSourceChange = (selectedOptions: DataSourceOption[]) => { setSelectedSources(selectedOptions); }; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 7fc088ddac3d..2e6ebf922424 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -24,6 +24,6 @@ export interface DataSourceSelectableProps { selectedSources: DataSourceOption[]; setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; - singleSelection: boolean; + singleSelection?: boolean; onFetchDataSetError: (error: Error) => void; } diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index c6ccb429128e..83d8cbc7aa13 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -50,8 +50,10 @@ export class DataSourceService { * @param datasources - An array of data sources to be registered. * @returns An array of registration results, one for each data source. */ - registerMultipleDataSources(datasources: DataSourceType[]) { - return datasources.map(this.registerDataSource); + async registerMultipleDataSources( + datasources: DataSourceType[] + ): Promise { + return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); } /** @@ -94,7 +96,7 @@ export class DataSourceService { const filteredDataSources: Record = {}; forEach(filters.names, (dsName) => { if (dsName in this.dataSources) { - filteredDataSources[dsName] = this.dataSources.dsName; + filteredDataSources[dsName] = this.dataSources[dsName]; } }); return filteredDataSources; diff --git a/src/plugins/discover/public/datasources/default_datasource.ts b/src/plugins/discover/public/datasources/default_datasource.ts index e5fdece61453..b3e50d8b1934 100644 --- a/src/plugins/discover/public/datasources/default_datasource.ts +++ b/src/plugins/discover/public/datasources/default_datasource.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { SavedObject } from '../../../saved_objects/public'; +import { IndexPatternSavedObjectAttrs } from '../../../data/common/index_patterns/index_patterns'; import { IndexPatternsContract } from '../../../data/public'; import { DataSource } from '../../../data/public'; @@ -16,7 +18,7 @@ interface DataSourceConfig { export class DefaultDslDataSource extends DataSource< any, any, - Promise[] | null | undefined>, + Promise> | null | undefined>, any, any > { @@ -29,11 +31,7 @@ export class DefaultDslDataSource extends DataSource< async getDataSet(dataSetParams?: any) { await this.indexPatterns.ensureDefaultIndexPattern(); - const ips = await this.indexPatterns.getCache(); - return { - ds: this, // datasource instance - data_sets: ips || [], // original dataset - }; + return await this.indexPatterns.getCache(); } async testConnection(): Promise { From e873130447b23fa55b4d7a29b5bdc7e722204937 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 00:52:19 +0000 Subject: [PATCH 19/56] add tests for datasource selectable Signed-off-by: Eric --- .../datasource_selectable.test.tsx | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx new file mode 100644 index 000000000000..9205f4652be2 --- /dev/null +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -0,0 +1,83 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { render, act } from '@testing-library/react'; +import { DataSourceSelectable } from './datasource_selectable'; +import { DataSourceType } from '../datasource_services'; + +describe('DataSourceSelectable', () => { + let dataSourcesMock; + let dataSourceOptionListMock; + let selectedSourcesMock; + let setSelectedSourcesMock; + let setDataSourceOptionListMock; + let onFetchDataSetErrorMock; + + beforeEach(() => { + dataSourcesMock = [ + ({ + getDataSet: jest.fn().mockResolvedValue([]), + getType: jest.fn().mockReturnValue('DEFAULT_INDEX_PATTERNS'), + getName: jest.fn().mockReturnValue('SomeName'), + } as unknown) as DataSourceType, + ]; + + dataSourceOptionListMock = []; + selectedSourcesMock = []; + setSelectedSourcesMock = jest.fn(); + setDataSourceOptionListMock = jest.fn(); + onFetchDataSetErrorMock = jest.fn(); + }); + + it('renders without crashing', () => { + render( + + ); + }); + + it('fetches data sets on mount', async () => { + await act(async () => { + render( + + ); + }); + + expect(dataSourcesMock[0].getDataSet).toHaveBeenCalled(); + }); + + it('handles data set fetch errors', async () => { + dataSourcesMock[0].getDataSet.mockRejectedValue(new Error('Fetch error')); + + await act(async () => { + render( + + ); + }); + + expect(onFetchDataSetErrorMock).toHaveBeenCalledWith(new Error('Fetch error')); + }); +}); From b54dedd0b7abe6ffd22465302ee753248c6d3cd3 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 02:52:47 +0000 Subject: [PATCH 20/56] added types/interfaces for sidebar selector, and add a couple of enhancements Signed-off-by: Eric --- .../datasource_selector/index.tsx | 2 +- src/plugins/data/public/index.ts | 2 ++ .../public/components/sidebar/index.tsx | 25 +++++++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx index 71dc982ee0a4..6d8004c1288f 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/index.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -5,4 +5,4 @@ export { DataSourceSelector } from './datasource_selector'; export { DataSourceSelectable } from './datasource_selectable'; -export { DataSourceSelectableProps, DataSourceGroup } from './types'; +export { DataSourceSelectableProps, DataSourceGroup, DataSourceOption } from './types'; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f57803bcbeb3..897695d85efd 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -519,4 +519,6 @@ export { DataSourceSelector, DataSourceSelectable, DataSourceSelectableProps, + DataSourceGroup, + DataSourceOption, } from './data_sources/datasource_selector'; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 3838d42eb13f..1ed698002018 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,19 +3,20 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { FC, useEffect, useState } from 'react'; +import React, { FC, useEffect, useState, useCallback } from 'react'; import { EuiSplitPanel, EuiPageSideBar } from '@elastic/eui'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; import { DataExplorerServices } from '../../types'; import { useTypedDispatch, useTypedSelector, setIndexPattern } from '../../utils/state_management'; -import { DataSourceSelectable } from '../../../../data/public'; +import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; +import { DataSourceOption } from '../../../../data/public/'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); const dispatch = useTypedDispatch(); - const [selectedSources, setSelectedSources] = useState([]); - const [dataSourceOptionList, setDataSourceOptionList] = useState([]); - const [activeDataSources, setActiveDataSources] = useState([]); + const [selectedSources, setSelectedSources] = useState([]); + const [dataSourceOptionList, setDataSourceOptionList] = useState([]); + const [activeDataSources, setActiveDataSources] = useState([]); const { services: { @@ -30,7 +31,7 @@ export const Sidebar: FC = ({ children }) => { const subscription = dataSources.dataSourceService.dataSources$.subscribe( (currentDataSources) => { if (isMounted) { - setActiveDataSources([...Object.values(currentDataSources)]); + setActiveDataSources(Object.values(currentDataSources)); } } ); @@ -41,7 +42,7 @@ export const Sidebar: FC = ({ children }) => { }; }, [indexPatterns, dataSources]); - const getMatchedOption = (dataSourceList, ipId: string) => { + const getMatchedOption = (dataSourceList: DataSourceGroup[], ipId: string) => { for (const dsGroup of dataSourceList) { return dsGroup.options.find((item) => { return item.value === ipId; @@ -56,7 +57,7 @@ export const Sidebar: FC = ({ children }) => { } }, [indexPatternId, activeDataSources, dataSourceOptionList]); - const handleSourceSelection = (selectedDataSources) => { + const handleSourceSelection = (selectedDataSources: DataSourceOption[]) => { // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource // will redirect user to Observability event explorer if (selectedDataSources[0].ds?.getType() !== 'Index patterns') { @@ -68,6 +69,13 @@ export const Sidebar: FC = ({ children }) => { dispatch(setIndexPattern(selectedDataSources[0].value)); }; + const handleDataSetFetchError = useCallback( + () => (error: Error) => { + toasts.addError(error, { title: `Data set fetching error: ${error}` }); + }, + [toasts] + ); + return ( @@ -78,6 +86,7 @@ export const Sidebar: FC = ({ children }) => { setDataSourceOptionList={setDataSourceOptionList} setSelectedSources={handleSourceSelection} selectedSources={selectedSources} + onFetchDataSetError={handleDataSetFetchError} /> {/* Date: Mon, 2 Oct 2023 03:00:12 +0000 Subject: [PATCH 21/56] remove pill effect Signed-off-by: Eric --- .../data/public/data_sources/datasource_selector/types.ts | 2 +- src/plugins/data_explorer/public/components/sidebar/index.tsx | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 2e6ebf922424..4dfaf53206c4 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -24,6 +24,6 @@ export interface DataSourceSelectableProps { selectedSources: DataSourceOption[]; setSelectedSources: (dataSourceOption: DataSourceOption[]) => void; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; - singleSelection?: boolean; + singleSelection?: boolean | { asPlainText: boolean }; onFetchDataSetError: (error: Error) => void; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 1ed698002018..f00913e3aea1 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -87,6 +87,7 @@ export const Sidebar: FC = ({ children }) => { setSelectedSources={handleSourceSelection} selectedSources={selectedSources} onFetchDataSetError={handleDataSetFetchError} + singleSelection={{ asPlainText: true }} /> {/* Date: Mon, 2 Oct 2023 04:23:49 +0000 Subject: [PATCH 22/56] change type for name display in selector for index patterns Signed-off-by: Eric --- src/plugins/discover/public/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index 20a3f3b53a0f..8b84b2fce499 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -378,7 +378,7 @@ export class DiscoverPlugin dataSourceService.registerDataSource( dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { name: DEFAULT_DATASOURCE_NAME, - type: INDEX_PATTERN_DATASOURCE_TYPE, + type: DEFAULT_DATASOURCE_TYPE, metadata: null, indexPatterns: plugins.data.indexPatterns, }) From 50ace8531a91672a1ee4f3419bfd95832cf12aed Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 19:45:26 +0000 Subject: [PATCH 23/56] remove legacy index selector Signed-off-by: Eric --- .../public/components/sidebar/index.tsx | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index f00913e3aea1..0925bba01f3c 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -89,44 +89,6 @@ export const Sidebar: FC = ({ children }) => { onFetchDataSetError={handleDataSetFetchError} singleSelection={{ asPlainText: true }} /> - {/* { - // TODO: There are many issues with this approach, but it's a start - // 1. Combo box can delete a selected index pattern. This should not be possible - // 2. Combo box is severely truncated. This should be fixed in the EUI component - // 3. The onchange can fire with a option that is not valid. discuss where to handle this. - // 4. value is optional. If the combobox needs to act as a slecet, this should be required. - const { value } = selected[0] || {}; - - if (!value) { - toasts.addWarning({ - id: 'index-pattern-not-found', - title: i18n.translate('dataExplorer.indexPatternError', { - defaultMessage: 'Index pattern not found', - }), - }); - return; - } - - dispatch(setIndexPattern(value)); - }} - /> */} - {/* Hidden for the 2.10 release of Data Explorer. Uncomment when Data explorer is released */} - {/* - { - dispatch(setView(e.target.value)); - }} - fullWidth - /> */} {children} From 8cd2fe303b8ed8246a167cbf0013728a64bc9b9f Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 19:59:07 +0000 Subject: [PATCH 24/56] rename datasource change handler Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 7 +++---- .../data/public/data_sources/datasource_selector/types.ts | 2 +- .../data_explorer/public/components/sidebar/index.tsx | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index b1628c751ec8..bbc679809d0c 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -23,7 +23,7 @@ export const DataSourceSelectable = ({ dataSources, // list of all available datasource connections. dataSourceOptionList, // combo box renderable option list derived from dataSources selectedSources, // current selected datasource in the form of [{ label: xxx, value: xxx }] - setSelectedSources, + onDataSourceSelect, setDataSourceOptionList, onFetchDataSetError, // onFetchDataSetError, Callback for handling dataset fetch errors. Ensure it's memoized. singleSelection = true, @@ -100,9 +100,8 @@ export const DataSourceSelectable = ({ .catch((e) => onFetchDataSetError(e)); }, [dataSources, setDataSourceOptionList, onFetchDataSetError]); - const handleSourceChange = (selectedOptions: DataSourceOption[]) => { - setSelectedSources(selectedOptions); - }; + const handleSourceChange = (selectedOptions: DataSourceOption[]) => + onDataSourceSelect(selectedOptions); return ( void; + onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; singleSelection?: boolean | { asPlainText: boolean }; onFetchDataSetError: (error: Error) => void; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 0925bba01f3c..55e1b7ba548f 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -84,7 +84,7 @@ export const Sidebar: FC = ({ children }) => { dataSources={activeDataSources} dataSourceOptionList={dataSourceOptionList} setDataSourceOptionList={setDataSourceOptionList} - setSelectedSources={handleSourceSelection} + onDataSourceSelect={handleSourceSelection} selectedSources={selectedSources} onFetchDataSetError={handleDataSetFetchError} singleSelection={{ asPlainText: true }} From dd4b542b754b08e392ff72362ff7930979dec933 Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 20:03:54 +0000 Subject: [PATCH 25/56] remove unused constants Signed-off-by: Eric --- .../data_sources/datasource_selector/constants.ts | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 src/plugins/data/public/data_sources/datasource_selector/constants.ts diff --git a/src/plugins/data/public/data_sources/datasource_selector/constants.ts b/src/plugins/data/public/data_sources/datasource_selector/constants.ts deleted file mode 100644 index 34cbf00147c7..000000000000 --- a/src/plugins/data/public/data_sources/datasource_selector/constants.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -export const SOURCE_PICKER_TITLE = 'available sources'; -export const SOURCE_PICKER_BTN_DEFAULT_TEXT = 'Select a datasource'; -export const SOURCE_PICKER_BTN_DEFAULT_WIDTH = '300px'; -export const SOURCE_PICKER_PANEL_DEFAULT_WIDTH = '500px'; -export const SOURCE_PICKER_PANEL_SEARCH_TEST_SUBJ = 'selectableDatasourcePanelSearch'; -export const SOURCE_PICKER_PANEL_TEST_SUBJ = 'selectableDatasourcePanel'; -export const SOURCE_PICKER_FOOTER_CANCEL_BTN_TEXT = 'Cancel'; -export const SOURCE_PICKER_FOOTER_SELECT_BTN_TEXT = 'Select'; -export const SOURCE_PICKER_FOOTER_SELECT_BTN_TEST_SUBJ = 'datasourcePickerSelect'; -export const SOURCE_PICKER_BTN_TEST_SUBJ = 'sourcePickerButton'; From 467b177e27d4b9c04ebde68f44b912147c0668ee Mon Sep 17 00:00:00 2001 From: Eric Date: Mon, 2 Oct 2023 20:07:48 +0000 Subject: [PATCH 26/56] fix a redirection bug Signed-off-by: Eric --- src/plugins/data_explorer/public/components/sidebar/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 55e1b7ba548f..33cc77360da7 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -60,7 +60,7 @@ export const Sidebar: FC = ({ children }) => { const handleSourceSelection = (selectedDataSources: DataSourceOption[]) => { // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource // will redirect user to Observability event explorer - if (selectedDataSources[0].ds?.getType() !== 'Index patterns') { + if (selectedDataSources[0].ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { return application.navigateToUrl( `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` ); From 20487e0026c6255d26822c6117eb0e48034d1905 Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 3 Oct 2023 00:54:01 +0000 Subject: [PATCH 27/56] add to change log Signed-off-by: Eric --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d414598dd927..4643a1347bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -661,6 +661,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### 📈 Features/Enhancements +- Add datasource selector to discover ([#5167](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5167)) - Add extension point in saved object management to register namespaces and show filter ([#2656](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2656)) - Add updated_at column to Saved Objects' tables ([#1218](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/1218)) - Change the links in the visualize plugin to use `href` rather than `onClick` ([#2395](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2395)) From 5532d3895c7449ba40c7e96e18ebc31aebb5c14a Mon Sep 17 00:00:00 2001 From: Eric Date: Tue, 3 Oct 2023 14:12:22 +0000 Subject: [PATCH 28/56] fix bg color issue in source selector Signed-off-by: Eric --- src/plugins/data_explorer/public/components/sidebar/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 33cc77360da7..2c01bf084c56 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -79,7 +79,7 @@ export const Sidebar: FC = ({ children }) => { return ( - + Date: Wed, 4 Oct 2023 00:19:56 +0000 Subject: [PATCH 29/56] address oui and missing guard Signed-off-by: Eric --- .../public/data_sources/datasource_selector/types.ts | 5 +++-- .../datasource_services/datasource_service.ts | 10 ++++------ .../data_explorer/public/components/sidebar/index.tsx | 4 ++++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index a919625cade6..7b68b664c57b 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { OuiComboBoxOptionOption } from '@elastic/eui'; +import { EuiComboBoxOptionOption } from '@elastic/eui'; import { DataSourceType } from '../datasource_services'; export interface DataSourceGroup { @@ -14,9 +14,10 @@ export interface DataSourceGroup { export interface DataSourceOption { label: string; value: string; + ds: DataSourceType; } -export type DataSourceOptionType = OuiComboBoxOptionOption; +export type DataSourceOptionType = EuiComboBoxOptionOption; export interface DataSourceSelectableProps { dataSources: DataSourceType[]; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 83d8cbc7aa13..b42c63deec6e 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -4,7 +4,6 @@ */ import { BehaviorSubject } from 'rxjs'; -import { isEmpty, forEach } from 'lodash'; import { DataSource, IDataSetParams, @@ -92,13 +91,12 @@ export class DataSourceService { * @returns A record of filtered data sources. */ getDataSources(filters?: IDataSourceFilters): Record { - if (!filters || isEmpty(filters.names)) return this.dataSources; - const filteredDataSources: Record = {}; - forEach(filters.names, (dsName) => { + if (!Array.isArray(filters?.names) || filters.names === 0) return this.dataSources; + return filters.names.reduce((filteredDataSources, dsName) => { if (dsName in this.dataSources) { filteredDataSources[dsName] = this.dataSources[dsName]; } - }); - return filteredDataSources; + return filteredDataSources; + }, {}); } } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 2c01bf084c56..c0cf9daf2f76 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -58,6 +58,10 @@ export const Sidebar: FC = ({ children }) => { }, [indexPatternId, activeDataSources, dataSourceOptionList]); const handleSourceSelection = (selectedDataSources: DataSourceOption[]) => { + if (selectedDataSources.length === 0) { + setSelectedSources(selectedDataSources); + return; + } // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource // will redirect user to Observability event explorer if (selectedDataSources[0].ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { From 78b8b8398721db1f029d6e034a3c0b8d5d5bfdcf Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 00:20:35 +0000 Subject: [PATCH 30/56] add test subj Signed-off-by: Eric --- .../data_sources/datasource_selector/datasource_selector.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx index b6501e4b054b..4247b36f20e4 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx @@ -24,6 +24,7 @@ export const DataSourceSelector = ({ return ( Date: Wed, 4 Oct 2023 00:50:48 +0000 Subject: [PATCH 31/56] i18 and datasource interface default return Signed-off-by: Eric --- .../data/public/data_sources/datasource/datasource.ts | 2 +- .../public/data_sources/datasource/factory.test.ts | 10 +++++----- .../datasource_selector/datasource_selector.tsx | 9 ++++++--- .../datasource_services/datasource_service.test.ts | 6 +++--- src/plugins/discover/common/index.ts | 10 ++++++++-- .../discover/public/datasources/default_datasource.ts | 8 ++++---- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index 8fe720b1cee0..23c72c3bbe15 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -68,5 +68,5 @@ export abstract class DataSource< * * @returns {ConnectionStatus | Promise} Status of the connection test. */ - abstract testConnection(): ConnectionStatus | Promise; + abstract testConnection(): ConnectionStatus | Promise; } diff --git a/src/plugins/data/public/data_sources/datasource/factory.test.ts b/src/plugins/data/public/data_sources/datasource/factory.test.ts index b80b927d9d11..0f9ea016748f 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.test.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.test.ts @@ -5,7 +5,7 @@ import { DataSourceFactory } from './factory'; import { DataSource } from './datasource'; -import { IndexPattern } from '../../index_patterns'; +import { IndexPattern, IndexPatternsService } from '../../index_patterns'; class MockDataSource extends DataSource { private readonly indexPatterns; @@ -19,7 +19,7 @@ class MockDataSource extends DataSource { name: string; type: string; metadata: any; - indexPatterns: IndexPattern; + indexPatterns: IndexPatternsService; }) { super(name, type, metadata); this.indexPatterns = indexPatterns; @@ -30,12 +30,12 @@ class MockDataSource extends DataSource { return await this.indexPatterns.getCache(); } - async testConnection(): Promise { - throw new Error('This operation is not supported for this class.'); + async testConnection(): Promise { + return true; } async runQuery(queryParams: any) { - return null; + return undefined; } } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx index 4247b36f20e4..94b2216008eb 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx @@ -4,8 +4,9 @@ */ import React from 'react'; +import { i18n } from '@osd/i18n'; import { EuiComboBox } from '@elastic/eui'; -import { DataSourceList, DataSourceOptionType } from './types'; +import { DataSourceOptionType } from './types'; export const DataSourceSelector = ({ dataSourceList, @@ -13,7 +14,7 @@ export const DataSourceSelector = ({ onDataSourceChange, singleSelection = true, }: { - dataSourceList: DataSourceList[]; + dataSourceList: DataSourceOptionType[]; selectedOptions: DataSourceOptionType[]; onDataSourceChange: (selectedDataSourceOptions: DataSourceOptionType[]) => void; singleSelection?: boolean; @@ -25,7 +26,9 @@ export const DataSourceSelector = ({ return ( { return await this.indexPatterns.getCache(); } - async testConnection(): Promise { - throw new Error('This operation is not supported for this class.'); + async testConnection(): Promise { + return true; } async runQuery(queryParams: any) { - return null; + return undefined; } } diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index fe7353217fef..cd40bbcd7602 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { i18n } from '@osd/i18n'; + export const PLUGIN_ID = 'discover'; export const DEFAULT_COLUMNS_SETTING = 'defaultColumns'; export const SAMPLE_SIZE_SETTING = 'discover:sampleSize'; @@ -16,5 +18,9 @@ export const CONTEXT_STEP_SETTING = 'context:step'; export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields'; export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch'; export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATASOURCE_NAME = 'OpenSearch Default'; -export const INDEX_PATTERN_DATASOURCE_TYPE = 'Index patterns'; +export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { + defaultMessage: 'OpenSearch Default', +}); +export const INDEX_PATTERN_DATASOURCE_TYPE = i18n.translate('data.datasource.type.indexPattern', { + defaultMessage: 'Index Patterns', +}); diff --git a/src/plugins/discover/public/datasources/default_datasource.ts b/src/plugins/discover/public/datasources/default_datasource.ts index b3e50d8b1934..d12a3d7fe8ff 100644 --- a/src/plugins/discover/public/datasources/default_datasource.ts +++ b/src/plugins/discover/public/datasources/default_datasource.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { SavedObject } from '../../../saved_objects/public'; +import { SavedObject } from '../../../../core/types'; import { IndexPatternSavedObjectAttrs } from '../../../data/common/index_patterns/index_patterns'; import { IndexPatternsContract } from '../../../data/public'; import { DataSource } from '../../../data/public'; @@ -34,11 +34,11 @@ export class DefaultDslDataSource extends DataSource< return await this.indexPatterns.getCache(); } - async testConnection(): Promise { - throw new Error('This operation is not supported for this class.'); + async testConnection(): Promise { + return true; } async runQuery(queryParams: any) { - return null; + return undefined; } } From 422da0b426a0667fb57166299b0eaec006399687 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 03:15:42 +0000 Subject: [PATCH 32/56] add default datasource tests Signed-off-by: Eric --- .../datasources/default_datasource.test.ts | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/plugins/discover/public/datasources/default_datasource.test.ts diff --git a/src/plugins/discover/public/datasources/default_datasource.test.ts b/src/plugins/discover/public/datasources/default_datasource.test.ts new file mode 100644 index 000000000000..776495b1f7d8 --- /dev/null +++ b/src/plugins/discover/public/datasources/default_datasource.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DefaultDslDataSource } from './default_datasource'; + +describe('DefaultDslDataSource', () => { + let indexPatternsMock; + + beforeEach(() => { + indexPatternsMock = { + ensureDefaultIndexPattern: jest.fn(), + getCache: jest.fn(), + }; + }); + + it('should ensure default index pattern and get cache', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + await dataSource.getDataSet(); + + expect(indexPatternsMock.ensureDefaultIndexPattern).toHaveBeenCalledTimes(1); + expect(indexPatternsMock.getCache).toHaveBeenCalledTimes(1); + }); + + it('should throw an error', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + await expect(dataSource.testConnection()).resolves.toBe(true); + }); + + it('should return null', async () => { + const dataSource = new DefaultDslDataSource({ + name: 'testName', + type: 'testType', + metadata: {}, + indexPatterns: indexPatternsMock, + }); + + const result = await dataSource.runQuery({}); + expect(result).toBeUndefined(); + }); +}); From f01ca6cb8c04f83c665fba0c9cec460dcb38b692 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 03:17:27 +0000 Subject: [PATCH 33/56] fix typo Signed-off-by: Eric --- src/plugins/data_explorer/public/components/sidebar/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index c0cf9daf2f76..8e7a4e4cc342 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -62,7 +62,7 @@ export const Sidebar: FC = ({ children }) => { setSelectedSources(selectedDataSources); return; } - // Temperary redirection solution for 2.11, where clicking non-index-pattern datasource + // Temporary redirection solution for 2.11, where clicking non-index-pattern datasource // will redirect user to Observability event explorer if (selectedDataSources[0].ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { return application.navigateToUrl( From 3be325f8c3d8cc1d63f5f0ddb24b8d38d7cb71c0 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 03:26:55 +0000 Subject: [PATCH 34/56] modify wording Signed-off-by: Eric --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df0beee21b04..b26192b24f36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -671,7 +671,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### 📈 Features/Enhancements -- Add datasource selector to discover ([#5167](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5167)) +- Add DataSource service and DataSourceSelector for multiple datasource support ([#5167](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5167)) - Add extension point in saved object management to register namespaces and show filter ([#2656](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2656)) - Add updated_at column to Saved Objects' tables ([#1218](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/1218)) - Change the links in the visualize plugin to use `href` rather than `onClick` ([#2395](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2395)) From dcc8b2f423fa8e980e934d99d052443545e5d140 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 17:32:08 +0000 Subject: [PATCH 35/56] add experimental annotation to datasource Signed-off-by: Eric --- src/plugins/data/public/data_sources/datasource/datasource.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index 23c72c3bbe15..5899abce8f17 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -50,6 +50,7 @@ export abstract class DataSource< * patterns for OpenSearch data source * * @returns {SourceDataSet} Dataset associated with the data source. + * @experimental */ abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; @@ -58,6 +59,7 @@ export abstract class DataSource< * Implementing classes need to provide the specific implementation. * * @returns {DataSourceQueryResult} Result from querying the data source. + * @experimental */ abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; @@ -67,6 +69,7 @@ export abstract class DataSource< * the connection status, typically indicating success or failure. * * @returns {ConnectionStatus | Promise} Status of the connection test. + * @experimental */ abstract testConnection(): ConnectionStatus | Promise; } From 8da7f5590e24ccc48c32e30ba5aeabb998394874 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 18:48:10 +0000 Subject: [PATCH 36/56] add type datasource filtering to remove type error hint Signed-off-by: Eric --- .../data_sources/datasource_services/datasource_service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index b42c63deec6e..c9ce27fb64a0 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -91,8 +91,8 @@ export class DataSourceService { * @returns A record of filtered data sources. */ getDataSources(filters?: IDataSourceFilters): Record { - if (!Array.isArray(filters?.names) || filters.names === 0) return this.dataSources; - return filters.names.reduce((filteredDataSources, dsName) => { + if (!Array.isArray(filters?.names) || filters.names.length === 0) return this.dataSources; + return filters.names.reduce>((filteredDataSources, dsName) => { if (dsName in this.dataSources) { filteredDataSources[dsName] = this.dataSources[dsName]; } From 55a1f3af96ab0a10e711b7365197203eef51d5d7 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 18:49:44 +0000 Subject: [PATCH 37/56] remove unused type Signed-off-by: Eric --- src/plugins/data_explorer/public/types.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/data_explorer/public/types.ts b/src/plugins/data_explorer/public/types.ts index 8bd3e51f3c3f..e1beff991443 100644 --- a/src/plugins/data_explorer/public/types.ts +++ b/src/plugins/data_explorer/public/types.ts @@ -32,12 +32,6 @@ export interface DataExplorerPluginStartDependencies { data: DataPublicPluginStart; } -export interface DataSourceStart { - dataSourceService: DataSourceService; - dataSourceFactory: DataSourceFactory; - DataSourceSelector: React.ComponentType; -} - export interface DataExplorerServices extends CoreStart { store?: Store; viewRegistry: ViewServiceStart; From 2227733ab872e5ca5b9165655ceccc6577f18af8 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 18:53:40 +0000 Subject: [PATCH 38/56] add 'type' to option type Signed-off-by: Eric --- .../data/public/data_sources/datasource_selector/types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 7b68b664c57b..c148d7adf15e 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -14,6 +14,7 @@ export interface DataSourceGroup { export interface DataSourceOption { label: string; value: string; + type: string; ds: DataSourceType; } From 64872bd87b0ef2f06983275ccbc2a614f89b0adc Mon Sep 17 00:00:00 2001 From: Ashwin P Chandran Date: Wed, 4 Oct 2023 20:07:12 +0000 Subject: [PATCH 39/56] Type fixes Signed-off-by: Ashwin P Chandran --- .../public/data_sources/datasource/factory.ts | 15 ++++-- .../datasource_service.test.ts | 16 +++---- .../datasource_services/datasource_service.ts | 46 ++++++++----------- .../data_sources/datasource_services/types.ts | 2 + .../default_datasource.test.ts | 7 +-- .../default_datasource}/default_datasource.ts | 10 ++-- .../data_sources/default_datasource}/index.ts | 0 .../register_default_datasource.ts | 27 +++++++++++ src/plugins/data/public/plugin.ts | 7 ++- src/plugins/discover/common/index.ts | 9 ---- src/plugins/discover/public/plugin.ts | 22 +-------- 11 files changed, 81 insertions(+), 80 deletions(-) rename src/plugins/{discover/public/datasources => data/public/data_sources/default_datasource}/default_datasource.test.ts (87%) rename src/plugins/{discover/public/datasources => data/public/data_sources/default_datasource}/default_datasource.ts (72%) rename src/plugins/{discover/public/datasources => data/public/data_sources/default_datasource}/index.ts (100%) create mode 100644 src/plugins/data/public/data_sources/register_default_datasource.ts diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts index c9ed8d91e041..917c9c8ac168 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -9,13 +9,22 @@ */ import { DataSourceType } from '../datasource_services'; +import { DataSource } from '../datasource'; + +type DataSourceClass< + MetaData = any, + SetParams = any, + DataSet = any, + QueryParams = any, + QueryResult = any +> = new (config: any) => DataSource; export class DataSourceFactory { // Holds the singleton instance of the DataSourceFactory. private static factory: DataSourceFactory; // A dictionary holding the data source type as the key and its corresponding class constructor as the value. - private dataSourceClasses: { [type: string]: new (config: any) => DataSourceType } = {}; + private dataSourceClasses: { [type: string]: DataSourceClass } = {}; /** * Private constructor to ensure only one instance of DataSourceFactory is created. @@ -39,10 +48,10 @@ export class DataSourceFactory { * If the type has already been registered, an error is thrown. * * @param {string} type - The identifier for the data source type. - * @param {new (config: any) => DataSourceType} dataSourceClass - The constructor of the data source class. + * @param {DataSourceClass} dataSourceClass - The constructor of the data source class. * @throws {Error} Throws an error if the data source type has already been registered. */ - registerDataSourceType(type: string, dataSourceClass: new (config: any) => DataSourceType): void { + registerDataSourceType(type: string, dataSourceClass: DataSourceClass): void { if (this.dataSourceClasses[type]) { throw new Error('This data source type has already been registered'); } diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index 811f5b488fbf..bff315d84ba7 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -4,30 +4,30 @@ */ import { DataSource } from '../datasource'; -import { IndexPattern } from '../../index_patterns'; +import { IndexPatternsService } from '../../index_patterns'; import { DataSourceService } from '../datasource_services'; class MockDataSource extends DataSource { - private readonly indexPatterns; + private readonly indexPattern; constructor({ name, type, metadata, - indexPatterns, + indexPattern, }: { name: string; type: string; metadata: any; - indexPatterns: IndexPattern; + indexPattern: IndexPatternsService; }) { super(name, type, metadata); - this.indexPatterns = indexPatterns; + this.indexPattern = indexPattern; } async getDataSet(dataSetParams?: any) { - await this.indexPatterns.ensureDefaultIndexPattern(); - return await this.indexPatterns.getCache(); + await this.indexPattern.ensureDefaultIndexPattern(); + return await this.indexPattern.getCache(); } async testConnection(): Promise { @@ -39,7 +39,7 @@ class MockDataSource extends DataSource { } } -const mockIndexPattern = {} as IndexPattern; +const mockIndexPattern = {} as IndexPatternsService; const mockConfig1 = { name: 'test_datasource1', diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index c9ce27fb64a0..37970e4f4679 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -4,33 +4,18 @@ */ import { BehaviorSubject } from 'rxjs'; -import { - DataSource, - IDataSetParams, - IDataSourceMetaData, - IDataSourceQueryParams, - IDataSourceQueryResult, - ISourceDataSet, -} from '../datasource'; import { IDataSourceFilters, IDataSourceRegisterationResult, DataSourceRegisterationError, + GenericDataSource, } from './types'; -export type DataSourceType = DataSource< - IDataSourceMetaData, - IDataSetParams, - ISourceDataSet, - IDataSourceQueryParams, - IDataSourceQueryResult ->; - export class DataSourceService { private static dataSourceService: DataSourceService; // A record to store all registered data sources, using the data source name as the key. - private dataSources: Record = {}; - private _dataSourcesSubject: BehaviorSubject>; + private dataSources: Record = {}; + private _dataSourcesSubject: BehaviorSubject>; private constructor() { this._dataSourcesSubject = new BehaviorSubject(this.dataSources); @@ -50,7 +35,7 @@ export class DataSourceService { * @returns An array of registration results, one for each data source. */ async registerMultipleDataSources( - datasources: DataSourceType[] + datasources: GenericDataSource[] ): Promise { return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); } @@ -63,7 +48,7 @@ export class DataSourceService { * @returns A registration result indicating success or failure. * @throws {DataSourceRegisterationError} Throws an error if a data source with the same name already exists. */ - async registerDataSource(ds: DataSourceType): Promise { + async registerDataSource(ds: GenericDataSource): Promise { const dsName = ds.getName(); if (dsName in this.dataSources) { throw new DataSourceRegisterationError( @@ -90,13 +75,18 @@ export class DataSourceService { * @param filters - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ - getDataSources(filters?: IDataSourceFilters): Record { - if (!Array.isArray(filters?.names) || filters.names.length === 0) return this.dataSources; - return filters.names.reduce>((filteredDataSources, dsName) => { - if (dsName in this.dataSources) { - filteredDataSources[dsName] = this.dataSources[dsName]; - } - return filteredDataSources; - }, {}); + getDataSources(filters?: IDataSourceFilters): Record { + if (!filters || !Array.isArray(filters?.names) || filters.names.length === 0) + return this.dataSources; + + return filters.names.reduce>( + (filteredDataSources, dsName) => { + if (dsName in this.dataSources) { + filteredDataSources[dsName] = this.dataSources[dsName]; + } + return filteredDataSources; + }, + {} as Record + ); } } diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index b0f4bf7d80d9..845329b61a61 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -45,3 +45,5 @@ export type DataSourceType = DataSource< IDataSourceQueryParams, IDataSourceQueryResult >; + +export type GenericDataSource = DataSource; diff --git a/src/plugins/discover/public/datasources/default_datasource.test.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts similarity index 87% rename from src/plugins/discover/public/datasources/default_datasource.test.ts rename to src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts index 776495b1f7d8..aedc6cd3853a 100644 --- a/src/plugins/discover/public/datasources/default_datasource.test.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.test.ts @@ -3,16 +3,17 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { IndexPatternsService } from '../../index_patterns'; import { DefaultDslDataSource } from './default_datasource'; describe('DefaultDslDataSource', () => { - let indexPatternsMock; + let indexPatternsMock: IndexPatternsService; beforeEach(() => { - indexPatternsMock = { + indexPatternsMock = ({ ensureDefaultIndexPattern: jest.fn(), getCache: jest.fn(), - }; + } as unknown) as IndexPatternsService; }); it('should ensure default index pattern and get cache', async () => { diff --git a/src/plugins/discover/public/datasources/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts similarity index 72% rename from src/plugins/discover/public/datasources/default_datasource.ts rename to src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index d12a3d7fe8ff..57c4d0382948 100644 --- a/src/plugins/discover/public/datasources/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -3,16 +3,16 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { SavedObject } from '../../../../core/types'; -import { IndexPatternSavedObjectAttrs } from '../../../data/common/index_patterns/index_patterns'; -import { IndexPatternsContract } from '../../../data/public'; -import { DataSource } from '../../../data/public'; +import { SavedObject } from '../../../../../core/types'; +import { IndexPatternsService } from '../../index_patterns'; +import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; +import { DataSource } from '../datasource'; interface DataSourceConfig { name: string; type: string; metadata: any; - indexPatterns: IndexPatternsContract; + indexPatterns: IndexPatternsService; } export class DefaultDslDataSource extends DataSource< diff --git a/src/plugins/discover/public/datasources/index.ts b/src/plugins/data/public/data_sources/default_datasource/index.ts similarity index 100% rename from src/plugins/discover/public/datasources/index.ts rename to src/plugins/data/public/data_sources/default_datasource/index.ts diff --git a/src/plugins/data/public/data_sources/register_default_datasource.ts b/src/plugins/data/public/data_sources/register_default_datasource.ts new file mode 100644 index 000000000000..8dece27e82eb --- /dev/null +++ b/src/plugins/data/public/data_sources/register_default_datasource.ts @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { i18n } from '@osd/i18n'; +import { DataPublicPluginStart } from '../types'; +import { DefaultDslDataSource } from './default_datasource'; + +export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; +export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { + defaultMessage: 'OpenSearch Default', +}); + +export const registerDefaultDatasource = (data: Omit) => { + // Datasources registrations for index patterns datasource + const { dataSourceService, dataSourceFactory } = data.dataSources; + dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); + dataSourceService.registerDataSource( + dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { + name: DEFAULT_DATASOURCE_NAME, + type: DEFAULT_DATASOURCE_TYPE, + metadata: null, + indexPatterns: data.indexPatterns, + }) + ); +}; diff --git a/src/plugins/data/public/plugin.ts b/src/plugins/data/public/plugin.ts index 23a29e4f8f8f..179b6c0a8c83 100644 --- a/src/plugins/data/public/plugin.ts +++ b/src/plugins/data/public/plugin.ts @@ -90,6 +90,7 @@ import { SavedObjectsClientPublicToCommon } from './index_patterns'; import { indexPatternLoad } from './index_patterns/expressions/load_index_pattern'; import { DataSourceService } from './data_sources/datasource_services'; import { DataSourceFactory } from './data_sources/datasource'; +import { registerDefaultDatasource } from './data_sources/register_default_datasource'; declare module '../../ui_actions/public' { export interface ActionContextMapping { @@ -234,6 +235,8 @@ export class DataPublicPlugin }, }; + registerDefaultDatasource(dataServices); + const SearchBar = createSearchBar({ core, data: dataServices, @@ -246,10 +249,6 @@ export class DataPublicPlugin IndexPatternSelect: createIndexPatternSelect(core.savedObjects.client), SearchBar, }, - dataSources: { - dataSourceService, - dataSourceFactory, - }, }; } diff --git a/src/plugins/discover/common/index.ts b/src/plugins/discover/common/index.ts index cd40bbcd7602..45887df880ae 100644 --- a/src/plugins/discover/common/index.ts +++ b/src/plugins/discover/common/index.ts @@ -3,8 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { i18n } from '@osd/i18n'; - export const PLUGIN_ID = 'discover'; export const DEFAULT_COLUMNS_SETTING = 'defaultColumns'; export const SAMPLE_SIZE_SETTING = 'discover:sampleSize'; @@ -17,10 +15,3 @@ export const CONTEXT_DEFAULT_SIZE_SETTING = 'context:defaultSize'; export const CONTEXT_STEP_SETTING = 'context:step'; export const CONTEXT_TIE_BREAKER_FIELDS_SETTING = 'context:tieBreakerFields'; export const MODIFY_COLUMNS_ON_SWITCH = 'discover:modifyColumnsOnSwitch'; -export const DEFAULT_DATASOURCE_TYPE = 'DEFAULT_INDEX_PATTERNS'; -export const DEFAULT_DATASOURCE_NAME = i18n.translate('data.datasource.type.openSearchDefault', { - defaultMessage: 'OpenSearch Default', -}); -export const INDEX_PATTERN_DATASOURCE_TYPE = i18n.translate('data.datasource.type.indexPattern', { - defaultMessage: 'Index Patterns', -}); diff --git a/src/plugins/discover/public/plugin.ts b/src/plugins/discover/public/plugin.ts index 8b84b2fce499..f8e0f254f925 100644 --- a/src/plugins/discover/public/plugin.ts +++ b/src/plugins/discover/public/plugin.ts @@ -60,12 +60,7 @@ import { DiscoverUrlGenerator, } from './url_generator'; import { SearchEmbeddableFactory } from './embeddable'; -import { - DEFAULT_DATASOURCE_TYPE, - PLUGIN_ID, - DEFAULT_DATASOURCE_NAME, - INDEX_PATTERN_DATASOURCE_TYPE, -} from '../common'; +import { PLUGIN_ID } from '../common'; import { DataExplorerPluginSetup } from '../../data_explorer/public'; import { registerFeature } from './register_feature'; import { @@ -74,7 +69,6 @@ import { getPreloadedState, } from './application/utils/state_management/discover_slice'; import { migrateUrlState } from './migrate_state'; -import { DefaultDslDataSource } from './datasources'; declare module '../../share/public' { export interface UrlGeneratorStateMapping { @@ -372,18 +366,6 @@ export class DiscoverPlugin this.initializeServices(); - // Datasources registrations for index patterns datasource - const { dataSourceService, dataSourceFactory } = plugins.data.dataSources; - dataSourceFactory.registerDataSourceType(DEFAULT_DATASOURCE_TYPE, DefaultDslDataSource); - dataSourceService.registerDataSource( - dataSourceFactory.getDataSourceInstance(DEFAULT_DATASOURCE_TYPE, { - name: DEFAULT_DATASOURCE_NAME, - type: DEFAULT_DATASOURCE_TYPE, - metadata: null, - indexPatterns: plugins.data.indexPatterns, - }) - ); - return { urlGenerator: this.urlGenerator, savedSearchLoader: createSavedSearchesLoader({ @@ -410,7 +392,7 @@ export class DiscoverPlugin const [coreStart, deps] = await core.getStartServices(); return { executeTriggerActions: deps.uiActions.executeTriggerActions, - isEditable: () => coreStart.application.capabilities.discover.save as boolean, + isEditable: () => coreStart.application.capabilities.discover?.save as boolean, }; }; From 00491ebf3115c825096976b1bec04850665445b8 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 20:34:55 +0000 Subject: [PATCH 40/56] cherry-pick type changes from remote and add partial fixes Signed-off-by: Eric --- .../datasource_selectable.tsx | 22 +++++++++++++++---- .../default_datasource/default_datasource.ts | 14 +++--------- src/plugins/data_explorer/public/types.ts | 8 +------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index bbc679809d0c..d4d38685f6d4 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -10,8 +10,10 @@ import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './ import { ISourceDataSet } from '../datasource/types'; import { IndexPattern } from '../../index_patterns'; +type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; + // Mapping between datasource type and its display name. -const DATASOURCE_TYPE_DISPLAY_NAME_MAP = { +const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { DEFAULT_INDEX_PATTERNS: 'Index patterns', s3glue: 'Amazon S3', spark: 'Spark', @@ -19,6 +21,11 @@ const DATASOURCE_TYPE_DISPLAY_NAME_MAP = { type DataSetType = string | IndexPattern; +interface DataSetWithSource { + ds: DataSourceType; + data_sets: string[] | IndexPattern[]; +} + export const DataSourceSelectable = ({ dataSources, // list of all available datasource connections. dataSourceOptionList, // combo box renderable option list derived from dataSources @@ -31,7 +38,7 @@ export const DataSourceSelectable = ({ // This effect fetches datasets and prepares the datasource list for UI rendering. useEffect(() => { // Fetches datasets for a given datasource and returns it along with the source. - const fetchDataSetWithSource = async (ds: DataSourceType): Promise => { + const fetchDataSetWithSource = async (ds: DataSourceType): Promise => { const dataSet = await ds.getDataSet(); return { ds, @@ -42,8 +49,15 @@ export const DataSourceSelectable = ({ // Map through all data sources and fetch their respective datasets. const fetchDataSets = () => dataSources.map((ds: DataSourceType) => fetchDataSetWithSource(ds)); - const isIndexPatterns = (dataset: string | IndexPattern) => - dataset.attributes?.title && dataset.id; + // const isIndexPatterns = (dataset: string | IndexPattern) => + // dataset.attributes?.title && dataset.id; + + const isIndexPatterns = (dataset: string | IndexPattern): boolean => { + if (typeof dataset === 'object' && 'attributes' in dataset) { + return Boolean(dataset.attributes?.title && dataset.id); + } + return false; + }; // Get the option format for the combo box from the dataSource and dataSet. const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index 57c4d0382948..b6d02539d7b2 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -3,17 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { SavedObject } from '../../../../../core/types'; -import { IndexPatternsService } from '../../index_patterns'; -import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; -import { DataSource } from '../datasource'; - -interface DataSourceConfig { - name: string; - type: string; - metadata: any; - indexPatterns: IndexPatternsService; -} +import { SavedObject } from '../../../../core/types'; +import { IndexPatternSavedObjectAttrs } from '../../../data/common/index_patterns/index_patterns'; +import { IndexPatternsContract, DataSourceConfig, DataSource } from '../../../data/public'; export class DefaultDslDataSource extends DataSource< any, diff --git a/src/plugins/data_explorer/public/types.ts b/src/plugins/data_explorer/public/types.ts index e1beff991443..5f677fb46cfd 100644 --- a/src/plugins/data_explorer/public/types.ts +++ b/src/plugins/data_explorer/public/types.ts @@ -8,13 +8,7 @@ import { EmbeddableStart } from '../../embeddable/public'; import { ExpressionsStart } from '../../expressions/public'; import { ViewServiceStart, ViewServiceSetup } from './services/view_service'; import { IOsdUrlStateStorage } from '../../opensearch_dashboards_utils/public'; -import { - DataPublicPluginSetup, - DataPublicPluginStart, - DataSourceFactory, - DataSourceService, - DataSourceSelectableProps, -} from '../../data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../data/public'; import { Store } from './utils/state_management'; export type DataExplorerPluginSetup = ViewServiceSetup; From ccbb92b680541dc56b5e44399ab26b3dacdfbe39 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 4 Oct 2023 20:56:34 +0000 Subject: [PATCH 41/56] add one type exports Signed-off-by: Eric --- .../data/public/data_sources/datasource/index.ts | 1 + .../data/public/data_sources/datasource/types.ts | 11 +++++++++-- src/plugins/data/public/index.ts | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index 0ab60d9b8d87..4908c97e502d 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -11,5 +11,6 @@ export { IDataSourceQueryParams, IDataSourceQueryResult, ConnectionStatus, + DataSourceConfig, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 26e5023c2a3d..a8e3376faece 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IndexPattern } from '../../index_patterns'; +import { IndexPattern, IndexPatternsContract } from '../../index_patterns'; import { DataSourceType } from '../datasource_services'; export interface IDataSourceMetaData { @@ -16,7 +16,7 @@ export interface IDataSourceGroup { export interface ISourceDataSet { ds: DataSourceType; - data_sets: string[] | IndexPattern; + data_sets: string[] | IndexPattern[]; } // to-dos: add common interfaces for datasource @@ -33,3 +33,10 @@ export interface ConnectionStatus { success: boolean; info: string; } + +export interface DataSourceConfig { + name: string; + type: string; + metadata: any; + indexPatterns: IndexPatternsContract; +} diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 897695d85efd..f2562f479989 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -507,6 +507,7 @@ export { ISourceDataSet, ConnectionStatus, DataSourceFactory, + DataSourceConfig, } from './data_sources/datasource'; export { DataSourceRegisterationError, From 09f7002348cf00e5f18f51f16e0c9bb2f20a2ad8 Mon Sep 17 00:00:00 2001 From: Ashwin P Chandran Date: Wed, 4 Oct 2023 22:53:40 +0000 Subject: [PATCH 42/56] remaining type errors fixed Signed-off-by: Ashwin P Chandran --- .../public/data_sources/datasource/index.ts | 1 + .../public/data_sources/datasource/types.ts | 11 ++- .../datasource_selectable.test.tsx | 23 ++--- .../datasource_selectable.tsx | 71 +++++++------ .../datasource_selector.test.tsx | 99 ------------------- .../datasource_selector.tsx | 39 -------- .../datasource_selector/index.tsx | 1 - .../data_sources/datasource_selector/types.ts | 16 +-- .../data_sources/datasource_services/index.ts | 1 + .../default_datasource/default_datasource.ts | 21 +++- src/plugins/data/public/index.ts | 1 - .../public/components/sidebar/index.tsx | 1 - 12 files changed, 81 insertions(+), 204 deletions(-) delete mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx delete mode 100644 src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx diff --git a/src/plugins/data/public/data_sources/datasource/index.ts b/src/plugins/data/public/data_sources/datasource/index.ts index 4908c97e502d..10af40fdcfa2 100644 --- a/src/plugins/data/public/data_sources/datasource/index.ts +++ b/src/plugins/data/public/data_sources/datasource/index.ts @@ -12,5 +12,6 @@ export { IDataSourceQueryResult, ConnectionStatus, DataSourceConfig, + IndexPatternOption, } from './types'; export { DataSourceFactory } from './factory'; diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index a8e3376faece..b09bd9745cf2 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,9 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { IndexPattern, IndexPatternsContract } from '../../index_patterns'; +import { IndexPatternsService } from '../../index_patterns'; import { DataSourceType } from '../datasource_services'; +export interface IndexPatternOption { + title: string; + id: string; +} + export interface IDataSourceMetaData { name: string; } @@ -16,7 +21,7 @@ export interface IDataSourceGroup { export interface ISourceDataSet { ds: DataSourceType; - data_sets: string[] | IndexPattern[]; + data_sets: string[] | IndexPatternOption[]; } // to-dos: add common interfaces for datasource @@ -38,5 +43,5 @@ export interface DataSourceConfig { name: string; type: string; metadata: any; - indexPatterns: IndexPatternsContract; + indexPatterns: IndexPatternsService; } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 9205f4652be2..4b0487f3194e 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -6,15 +6,16 @@ import React from 'react'; import { render, act } from '@testing-library/react'; import { DataSourceSelectable } from './datasource_selectable'; -import { DataSourceType } from '../datasource_services'; +import { DataSourceType, GenericDataSource } from '../datasource_services'; +import { DataSourceGroup, DataSourceOption } from './types'; describe('DataSourceSelectable', () => { - let dataSourcesMock; - let dataSourceOptionListMock; - let selectedSourcesMock; - let setSelectedSourcesMock; - let setDataSourceOptionListMock; - let onFetchDataSetErrorMock; + let dataSourcesMock: GenericDataSource[]; + let dataSourceOptionListMock: DataSourceGroup[]; + let selectedSourcesMock: DataSourceOption[]; + let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); + let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); + let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); beforeEach(() => { dataSourcesMock = [ @@ -38,7 +39,7 @@ describe('DataSourceSelectable', () => { dataSources={dataSourcesMock} dataSourceOptionList={dataSourceOptionListMock} selectedSources={selectedSourcesMock} - setSelectedSources={setSelectedSourcesMock} + onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onFetchDataSetError={onFetchDataSetErrorMock} /> @@ -52,7 +53,7 @@ describe('DataSourceSelectable', () => { dataSources={dataSourcesMock} dataSourceOptionList={dataSourceOptionListMock} selectedSources={selectedSourcesMock} - setSelectedSources={setSelectedSourcesMock} + onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onFetchDataSetError={onFetchDataSetErrorMock} /> @@ -63,7 +64,7 @@ describe('DataSourceSelectable', () => { }); it('handles data set fetch errors', async () => { - dataSourcesMock[0].getDataSet.mockRejectedValue(new Error('Fetch error')); + (dataSourcesMock[0].getDataSet as jest.Mock).mockRejectedValue(new Error('Fetch error')); await act(async () => { render( @@ -71,7 +72,7 @@ describe('DataSourceSelectable', () => { dataSources={dataSourcesMock} dataSourceOptionList={dataSourceOptionListMock} selectedSources={selectedSourcesMock} - setSelectedSources={setSelectedSourcesMock} + onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} onFetchDataSetError={onFetchDataSetErrorMock} /> diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index d4d38685f6d4..57e6d86ece12 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -4,11 +4,11 @@ */ import React, { useEffect } from 'react'; -import { DataSourceSelector } from './datasource_selector'; -import { DataSourceType } from '../datasource_services'; -import { DataSourceGroup, DataSourceOption, DataSourceSelectableProps } from './types'; -import { ISourceDataSet } from '../datasource/types'; -import { IndexPattern } from '../../index_patterns'; +import { i18n } from '@osd/i18n'; +import { EuiComboBox } from '@elastic/eui'; +import { DataSourceType, GenericDataSource } from '../datasource_services'; +import { DataSourceGroup, DataSourceSelectableProps } from './types'; +import { ISourceDataSet, IndexPatternOption } from '../datasource'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; @@ -19,11 +19,11 @@ const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { spark: 'Spark', }; -type DataSetType = string | IndexPattern; +type DataSetType = string | IndexPatternOption; interface DataSetWithSource { - ds: DataSourceType; - data_sets: string[] | IndexPattern[]; + ds: GenericDataSource; + data_sets: string[] | IndexPatternOption[]; } export const DataSourceSelectable = ({ @@ -38,7 +38,7 @@ export const DataSourceSelectable = ({ // This effect fetches datasets and prepares the datasource list for UI rendering. useEffect(() => { // Fetches datasets for a given datasource and returns it along with the source. - const fetchDataSetWithSource = async (ds: DataSourceType): Promise => { + const fetchDataSetWithSource = async (ds: GenericDataSource): Promise => { const dataSet = await ds.getDataSet(); return { ds, @@ -49,14 +49,10 @@ export const DataSourceSelectable = ({ // Map through all data sources and fetch their respective datasets. const fetchDataSets = () => dataSources.map((ds: DataSourceType) => fetchDataSetWithSource(ds)); - // const isIndexPatterns = (dataset: string | IndexPattern) => - // dataset.attributes?.title && dataset.id; + const isIndexPatterns = (dataset: string | IndexPatternOption): boolean => { + if (typeof dataset === 'string') return false; - const isIndexPatterns = (dataset: string | IndexPattern): boolean => { - if (typeof dataset === 'object' && 'attributes' in dataset) { - return Boolean(dataset.attributes?.title && dataset.id); - } - return false; + return dataset.title !== undefined && dataset.id !== undefined; }; // Get the option format for the combo box from the dataSource and dataSet. @@ -67,10 +63,11 @@ export const DataSourceSelectable = ({ ds: dataSource, }; if (isIndexPatterns(dataSet)) { + const ip = dataSet as IndexPatternOption; return { ...optionContent, - label: dataSet.attributes.title, - value: dataSet.id, + label: ip.title, + value: ip.id, }; } return { @@ -84,23 +81,21 @@ export const DataSourceSelectable = ({ const getSourceList = (allDataSets: ISourceDataSet[]) => { const finalList = [] as DataSourceGroup[]; allDataSets.forEach((curDataSet) => { - const existingGroup = finalList.find( - (item) => item.label === DATASOURCE_TYPE_DISPLAY_NAME_MAP[curDataSet.ds.getType()] + const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; + const groupName = DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || 'Default Group'; + + const existingGroup = finalList.find((item) => item.label === groupName); + const mappedOptions = curDataSet.data_sets?.map((dataSet) => + getSourceOptions(curDataSet.ds, dataSet) ); + // check if add new datasource group or add to existing one if (existingGroup) { - existingGroup.options = [ - ...existingGroup.options, - ...curDataSet.data_sets?.map((dataSet: DataSetType) => - getSourceOptions(curDataSet.ds, dataSet) - ), - ]; + existingGroup.options.push(...mappedOptions); } else { finalList.push({ - label: DATASOURCE_TYPE_DISPLAY_NAME_MAP[curDataSet.ds.getType()] || 'Default Group', - options: curDataSet.data_sets?.map((dataSet: DataSetType) => - getSourceOptions(curDataSet.ds, dataSet) - ), + label: groupName, + options: mappedOptions, }); } }); @@ -114,15 +109,19 @@ export const DataSourceSelectable = ({ .catch((e) => onFetchDataSetError(e)); }, [dataSources, setDataSourceOptionList, onFetchDataSetError]); - const handleSourceChange = (selectedOptions: DataSourceOption[]) => - onDataSourceSelect(selectedOptions); + const handleSourceChange = (selectedOptions: any) => onDataSourceSelect(selectedOptions); return ( - ); }; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx deleted file mode 100644 index a71c3b025be6..000000000000 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.test.tsx +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { render, fireEvent } from '@testing-library/react'; -import { DataSourceSelector } from './datasource_selector'; - -describe('DataSourceSelector', () => { - const mockOnDataSourceChange = jest.fn(); - - const sampleDataSources = [ - { - label: 'Index patterns', - options: [ - { label: 'sample_log1', value: 'sample_log1' }, - { label: 'sample_log2', value: 'sample_log2' }, - ], - }, - { - label: 'EMR', - options: [{ label: 'EMR_cluster', value: 'EMR_cluster' }], - }, - ]; - - const selectedSource = [{ label: 'sample_log1', value: 'sample_log1' }]; - - it('renders without crashing', () => { - const { getByText } = render( - - ); - - expect(getByText('sample_log1')).toBeInTheDocument(); - }); - - it('triggers onDataSourceChange when a data source is selected', () => { - const { getByTestId, getByText } = render( - - ); - - fireEvent.click(getByTestId('comboBoxToggleListButton')); - fireEvent.click(getByText('sample_log2')); - - expect(mockOnDataSourceChange).toHaveBeenCalledWith([ - { label: 'sample_log2', value: 'sample_log2' }, - ]); - }); - - it('has singleSelection set to true by default', () => { - const { rerender } = render( - - ); - - let comboBox = document.querySelector('[data-test-subj="comboBoxInput"]'); - expect(comboBox).toBeInTheDocument(); - - rerender( - - ); - - comboBox = document.querySelector('[data-test-subj="comboBoxInput"]'); - expect(comboBox).toBeInTheDocument(); - }); - - it('renders all data source options', () => { - const { getByText, getByTestId } = render( - - ); - - fireEvent.click(getByTestId('comboBoxToggleListButton')); - - expect(getByText('Index patterns')).toBeInTheDocument(); - expect(getByText('sample_log2')).toBeInTheDocument(); - expect(getByText('EMR')).toBeInTheDocument(); - expect(getByText('EMR_cluster')).toBeInTheDocument(); - }); -}); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx deleted file mode 100644 index 94b2216008eb..000000000000 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selector.tsx +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -import React from 'react'; -import { i18n } from '@osd/i18n'; -import { EuiComboBox } from '@elastic/eui'; -import { DataSourceOptionType } from './types'; - -export const DataSourceSelector = ({ - dataSourceList, - selectedOptions, - onDataSourceChange, - singleSelection = true, -}: { - dataSourceList: DataSourceOptionType[]; - selectedOptions: DataSourceOptionType[]; - onDataSourceChange: (selectedDataSourceOptions: DataSourceOptionType[]) => void; - singleSelection?: boolean; -}) => { - const onDataSourceSelectionChange = (selectedDataSourceOptions: DataSourceOptionType[]) => { - onDataSourceChange(selectedDataSourceOptions); - }; - - return ( - - ); -}; diff --git a/src/plugins/data/public/data_sources/datasource_selector/index.tsx b/src/plugins/data/public/data_sources/datasource_selector/index.tsx index 6d8004c1288f..763c83069a6f 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/index.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/index.tsx @@ -3,6 +3,5 @@ * SPDX-License-Identifier: Apache-2.0 */ -export { DataSourceSelector } from './datasource_selector'; export { DataSourceSelectable } from './datasource_selectable'; export { DataSourceSelectableProps, DataSourceGroup, DataSourceOption } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index c148d7adf15e..274c8cfd83b9 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -3,8 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiComboBoxOptionOption } from '@elastic/eui'; -import { DataSourceType } from '../datasource_services'; +import { EuiComboBoxOptionOption, EuiComboBoxSingleSelectionShape } from '@elastic/eui'; +import { GenericDataSource } from '../datasource_services'; export interface DataSourceGroup { label: string; @@ -15,17 +15,17 @@ export interface DataSourceOption { label: string; value: string; type: string; - ds: DataSourceType; + ds: GenericDataSource; } -export type DataSourceOptionType = EuiComboBoxOptionOption; +export type DataSourceOptionType = EuiComboBoxOptionOption; export interface DataSourceSelectableProps { - dataSources: DataSourceType[]; + dataSources: GenericDataSource[]; + onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; + singleSelection?: boolean | EuiComboBoxSingleSelectionShape; + onFetchDataSetError: (error: Error) => void; dataSourceOptionList: DataSourceGroup[]; selectedSources: DataSourceOption[]; - onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; - singleSelection?: boolean | { asPlainText: boolean }; - onFetchDataSetError: (error: Error) => void; } diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts index fdd07ba9dcc9..eb9f40c9e2a7 100644 --- a/src/plugins/data/public/data_sources/datasource_services/index.ts +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -9,4 +9,5 @@ export { IDataSourceRegisterationResult, DataSourceRegisterationError, DataSourceType, + GenericDataSource, } from './types'; diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index b6d02539d7b2..eb6fd2de582d 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -3,14 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { SavedObject } from '../../../../core/types'; -import { IndexPatternSavedObjectAttrs } from '../../../data/common/index_patterns/index_patterns'; -import { IndexPatternsContract, DataSourceConfig, DataSource } from '../../../data/public'; +import { SavedObject } from '../../../../../core/public'; +import { IndexPatternSavedObjectAttrs } from '../../index_patterns/index_patterns'; +import { DataSource, DataSourceConfig, IndexPatternOption } from '../datasource'; export class DefaultDslDataSource extends DataSource< any, any, - Promise> | null | undefined>, + Promise, any, any > { @@ -23,7 +23,18 @@ export class DefaultDslDataSource extends DataSource< async getDataSet(dataSetParams?: any) { await this.indexPatterns.ensureDefaultIndexPattern(); - return await this.indexPatterns.getCache(); + const savedObjectLst = await this.indexPatterns.getCache(); + + if (!savedObjectLst) { + return undefined; + } + + return savedObjectLst.map((savedObject: SavedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + }; + }); } async testConnection(): Promise { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f2562f479989..16cebbcc78ec 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -517,7 +517,6 @@ export { IDataSourceRegisterationResult, } from './data_sources/datasource_services'; export { - DataSourceSelector, DataSourceSelectable, DataSourceSelectableProps, DataSourceGroup, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 8e7a4e4cc342..e61194d66a4c 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -91,7 +91,6 @@ export const Sidebar: FC = ({ children }) => { onDataSourceSelect={handleSourceSelection} selectedSources={selectedSources} onFetchDataSetError={handleDataSetFetchError} - singleSelection={{ asPlainText: true }} /> From 811347c12329ccd1d4c86c106247ad140d281169 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 02:39:52 +0000 Subject: [PATCH 43/56] addressing comments Signed-off-by: Eric --- .../datasource_services/datasource_service.ts | 3 +-- .../data_sources/default_datasource/default_datasource.ts | 2 +- .../data_explorer/public/components/sidebar/index.tsx | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 37970e4f4679..3b0efcb4d660 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -76,8 +76,7 @@ export class DataSourceService { * @returns A record of filtered data sources. */ getDataSources(filters?: IDataSourceFilters): Record { - if (!filters || !Array.isArray(filters?.names) || filters.names.length === 0) - return this.dataSources; + if (!Array.isArray(filters?.names) || filters.names.length === 0) return this.dataSources; return filters.names.reduce>( (filteredDataSources, dsName) => { diff --git a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts index eb6fd2de582d..c3b5d2a4cf99 100644 --- a/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts +++ b/src/plugins/data/public/data_sources/default_datasource/default_datasource.ts @@ -25,7 +25,7 @@ export class DefaultDslDataSource extends DataSource< await this.indexPatterns.ensureDefaultIndexPattern(); const savedObjectLst = await this.indexPatterns.getCache(); - if (!savedObjectLst) { + if (!Array.isArray(savedObjectLst)) { return undefined; } diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index e61194d66a4c..154040b31780 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -44,10 +44,10 @@ export const Sidebar: FC = ({ children }) => { const getMatchedOption = (dataSourceList: DataSourceGroup[], ipId: string) => { for (const dsGroup of dataSourceList) { - return dsGroup.options.find((item) => { - return item.value === ipId; - }); + const matchedOption = dsGroup.options.find((item) => item.value === ipId); + if (matchedOption !== undefined) return matchedOption; } + return undefined; }; useEffect(() => { @@ -64,7 +64,7 @@ export const Sidebar: FC = ({ children }) => { } // Temporary redirection solution for 2.11, where clicking non-index-pattern datasource // will redirect user to Observability event explorer - if (selectedDataSources[0].ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { + if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { return application.navigateToUrl( `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` ); From e8f219b3febaaa5d5d7ce21cd9c39dc560268e0e Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 03:34:39 +0000 Subject: [PATCH 44/56] address dedup Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 57e6d86ece12..7256ee56fb8b 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -91,7 +91,11 @@ export const DataSourceSelectable = ({ // check if add new datasource group or add to existing one if (existingGroup) { - existingGroup.options.push(...mappedOptions); + const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); + const nonDuplicateOptions = mappedOptions.filter( + (opt) => !existingOptionIds.has(opt.label) + ); + existingGroup.options.push(...nonDuplicateOptions); } else { finalList.push({ label: groupName, From 3642ef5938360b1c7d1b0d064b3d3b1f1a7b8b9a Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Thu, 5 Oct 2023 18:48:52 +0000 Subject: [PATCH 45/56] refactor datasource_selectable to address comments Signed-off-by: Joshua Li --- .../public/data_sources/datasource/types.ts | 2 +- .../datasource_selectable.tsx | 153 +++++++++--------- 2 files changed, 77 insertions(+), 78 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index b09bd9745cf2..8d7aa853fb07 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -21,7 +21,7 @@ export interface IDataSourceGroup { export interface ISourceDataSet { ds: DataSourceType; - data_sets: string[] | IndexPatternOption[]; + data_sets: Array; } // to-dos: add common interfaces for datasource diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 7256ee56fb8b..5b40e01478af 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -3,12 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { useEffect } from 'react'; -import { i18n } from '@osd/i18n'; import { EuiComboBox } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import React, { useEffect } from 'react'; +import { ISourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; -import { ISourceDataSet, IndexPatternOption } from '../datasource'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; @@ -19,94 +19,93 @@ const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { spark: 'Spark', }; -type DataSetType = string | IndexPatternOption; +type DataSetType = ISourceDataSet['data_sets'][number]; interface DataSetWithSource { ds: GenericDataSource; - data_sets: string[] | IndexPatternOption[]; + data_sets: DataSetType[]; } +// Fetches data sets for a given datasource and returns it along with the source. +const fetchDataSetWithSource = async (ds: GenericDataSource): Promise => { + const dataSet = await ds.getDataSet(); + return { + ds, + data_sets: dataSet, + }; +}; + +// Map through all data sources and fetch their respective data sets. +const fetchDataSets = (dataSources: GenericDataSource[]) => + dataSources.map((ds) => fetchDataSetWithSource(ds)); + +const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { + if (typeof dataSet === 'string') return false; + + return !!(dataSet.title && dataSet.id); +}; + +// Get the option format for the combo box from the dataSource and dataSet. +const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { + const optionContent = { + type: dataSource.getType(), + name: dataSource.getName(), + ds: dataSource, + }; + if (isIndexPatterns(dataSet)) { + const ip = dataSet as IndexPatternOption; + return { + ...optionContent, + label: ip.title, + value: ip.id, + }; + } + return { + ...optionContent, + label: dataSource.getName(), + value: dataSource.getName(), + }; +}; + +// Convert fetched data sets into a structured format suitable for selector rendering. +const getSourceList = (allDataSets: ISourceDataSet[]) => { + const finalList = [] as DataSourceGroup[]; + allDataSets.forEach((curDataSet) => { + const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; + const groupName = DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || 'Default Group'; + + const existingGroup = finalList.find((item) => item.label === groupName); + const mappedOptions = curDataSet.data_sets?.map((dataSet) => + getSourceOptions(curDataSet.ds, dataSet) + ); + + // check if add new datasource group or add to existing one + if (existingGroup) { + const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); + const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + existingGroup.options.push(...nonDuplicateOptions); + } else { + finalList.push({ + label: groupName, + options: mappedOptions, + }); + } + }); + return finalList; +}; + export const DataSourceSelectable = ({ dataSources, // list of all available datasource connections. dataSourceOptionList, // combo box renderable option list derived from dataSources selectedSources, // current selected datasource in the form of [{ label: xxx, value: xxx }] onDataSourceSelect, setDataSourceOptionList, - onFetchDataSetError, // onFetchDataSetError, Callback for handling dataset fetch errors. Ensure it's memoized. + onFetchDataSetError, // onFetchDataSetError, Callback for handling data set fetch errors. Ensure it's memoized. singleSelection = true, }: DataSourceSelectableProps) => { - // This effect fetches datasets and prepares the datasource list for UI rendering. + // This effect fetches data sets and prepares the datasource list for UI rendering. useEffect(() => { - // Fetches datasets for a given datasource and returns it along with the source. - const fetchDataSetWithSource = async (ds: GenericDataSource): Promise => { - const dataSet = await ds.getDataSet(); - return { - ds, - data_sets: dataSet, - }; - }; - - // Map through all data sources and fetch their respective datasets. - const fetchDataSets = () => dataSources.map((ds: DataSourceType) => fetchDataSetWithSource(ds)); - - const isIndexPatterns = (dataset: string | IndexPatternOption): boolean => { - if (typeof dataset === 'string') return false; - - return dataset.title !== undefined && dataset.id !== undefined; - }; - - // Get the option format for the combo box from the dataSource and dataSet. - const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { - const optionContent = { - type: dataSource.getType(), - name: dataSource.getName(), - ds: dataSource, - }; - if (isIndexPatterns(dataSet)) { - const ip = dataSet as IndexPatternOption; - return { - ...optionContent, - label: ip.title, - value: ip.id, - }; - } - return { - ...optionContent, - label: dataSource.getName(), - value: dataSource.getName(), - }; - }; - - // Convert fetched datasets into a structured format suitable for selector rendering. - const getSourceList = (allDataSets: ISourceDataSet[]) => { - const finalList = [] as DataSourceGroup[]; - allDataSets.forEach((curDataSet) => { - const typeKey = curDataSet.ds.getType() as DataSourceTypeKey; - const groupName = DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || 'Default Group'; - - const existingGroup = finalList.find((item) => item.label === groupName); - const mappedOptions = curDataSet.data_sets?.map((dataSet) => - getSourceOptions(curDataSet.ds, dataSet) - ); - - // check if add new datasource group or add to existing one - if (existingGroup) { - const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); - const nonDuplicateOptions = mappedOptions.filter( - (opt) => !existingOptionIds.has(opt.label) - ); - existingGroup.options.push(...nonDuplicateOptions); - } else { - finalList.push({ - label: groupName, - options: mappedOptions, - }); - } - }); - return finalList; - }; - - Promise.all(fetchDataSets()) + Promise.all(fetchDataSets(dataSources)) .then((results) => { setDataSourceOptionList(getSourceList(results)); }) From a27d7e0fad8494491cff911190ca5b78f895be01 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Thu, 5 Oct 2023 20:22:11 +0000 Subject: [PATCH 46/56] remove unnecessary optional chaining Signed-off-by: Joshua Li --- .../data_sources/datasource_selector/datasource_selectable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 5b40e01478af..5b7fb8bf323c 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -75,7 +75,7 @@ const getSourceList = (allDataSets: ISourceDataSet[]) => { const groupName = DATASOURCE_TYPE_DISPLAY_NAME_MAP[typeKey] || 'Default Group'; const existingGroup = finalList.find((item) => item.label === groupName); - const mappedOptions = curDataSet.data_sets?.map((dataSet) => + const mappedOptions = curDataSet.data_sets.map((dataSet) => getSourceOptions(curDataSet.ds, dataSet) ); From d56bc1d4a19739b5df8d7d604231e7a8e8adef12 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Thu, 5 Oct 2023 21:28:04 +0000 Subject: [PATCH 47/56] refactor variable names Signed-off-by: Joshua Li --- .../data_sources/datasource/datasource.ts | 2 +- .../datasource_selectable.test.tsx | 6 +-- .../datasource_selectable.tsx | 27 ++++------ .../data_sources/datasource_selector/types.ts | 6 +-- .../datasource_service.test.ts | 4 +- .../datasource_services/datasource_service.ts | 51 +++++++++---------- .../data_sources/datasource_services/index.ts | 6 +-- .../data_sources/datasource_services/types.ts | 6 +-- src/plugins/data/public/index.ts | 6 +-- .../public/components/sidebar/index.tsx | 22 +++++--- 10 files changed, 65 insertions(+), 71 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index 5899abce8f17..dda3f02255f4 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -36,7 +36,7 @@ export abstract class DataSource< return this.type; } - getMetadata(): DataSourceMetaData { + getMetadata() { return this.metadata; } diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 4b0487f3194e..bd424211325e 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -41,7 +41,7 @@ describe('DataSourceSelectable', () => { selectedSources={selectedSourcesMock} onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} - onFetchDataSetError={onFetchDataSetErrorMock} + onGetDataSetError={onFetchDataSetErrorMock} /> ); }); @@ -55,7 +55,7 @@ describe('DataSourceSelectable', () => { selectedSources={selectedSourcesMock} onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} - onFetchDataSetError={onFetchDataSetErrorMock} + onGetDataSetError={onFetchDataSetErrorMock} /> ); }); @@ -74,7 +74,7 @@ describe('DataSourceSelectable', () => { selectedSources={selectedSourcesMock} onDataSourceSelect={setSelectedSourcesMock} setDataSourceOptionList={setDataSourceOptionListMock} - onFetchDataSetError={onFetchDataSetErrorMock} + onGetDataSetError={onFetchDataSetErrorMock} /> ); }); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 5b7fb8bf323c..645b96e9d1ee 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -21,13 +21,8 @@ const DATASOURCE_TYPE_DISPLAY_NAME_MAP: Record = { type DataSetType = ISourceDataSet['data_sets'][number]; -interface DataSetWithSource { - ds: GenericDataSource; - data_sets: DataSetType[]; -} - -// Fetches data sets for a given datasource and returns it along with the source. -const fetchDataSetWithSource = async (ds: GenericDataSource): Promise => { +// Get data sets for a given datasource and returns it along with the source. +const getDataSetWithSource = async (ds: GenericDataSource): Promise => { const dataSet = await ds.getDataSet(); return { ds, @@ -35,9 +30,9 @@ const fetchDataSetWithSource = async (ds: GenericDataSource): Promise - dataSources.map((ds) => fetchDataSetWithSource(ds)); +// Map through all data sources and get their respective data sets. +const getDataSets = (dataSources: GenericDataSource[]) => + dataSources.map((ds) => getDataSetWithSource(ds)); const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { if (typeof dataSet === 'string') return false; @@ -67,7 +62,7 @@ const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { }; }; -// Convert fetched data sets into a structured format suitable for selector rendering. +// Convert data sets into a structured format suitable for selector rendering. const getSourceList = (allDataSets: ISourceDataSet[]) => { const finalList = [] as DataSourceGroup[]; allDataSets.forEach((curDataSet) => { @@ -100,17 +95,17 @@ export const DataSourceSelectable = ({ selectedSources, // current selected datasource in the form of [{ label: xxx, value: xxx }] onDataSourceSelect, setDataSourceOptionList, - onFetchDataSetError, // onFetchDataSetError, Callback for handling data set fetch errors. Ensure it's memoized. + onGetDataSetError, // onGetDataSetError, Callback for handling get data set errors. Ensure it's memoized. singleSelection = true, }: DataSourceSelectableProps) => { - // This effect fetches data sets and prepares the datasource list for UI rendering. + // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { - Promise.all(fetchDataSets(dataSources)) + Promise.all(getDataSets(dataSources)) .then((results) => { setDataSourceOptionList(getSourceList(results)); }) - .catch((e) => onFetchDataSetError(e)); - }, [dataSources, setDataSourceOptionList, onFetchDataSetError]); + .catch((e) => onGetDataSetError(e)); + }, [dataSources, setDataSourceOptionList, onGetDataSetError]); const handleSourceChange = (selectedOptions: any) => onDataSourceSelect(selectedOptions); diff --git a/src/plugins/data/public/data_sources/datasource_selector/types.ts b/src/plugins/data/public/data_sources/datasource_selector/types.ts index 274c8cfd83b9..2d644a551759 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/types.ts +++ b/src/plugins/data/public/data_sources/datasource_selector/types.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { EuiComboBoxOptionOption, EuiComboBoxSingleSelectionShape } from '@elastic/eui'; +import { EuiComboBoxSingleSelectionShape } from '@elastic/eui'; import { GenericDataSource } from '../datasource_services'; export interface DataSourceGroup { @@ -18,13 +18,11 @@ export interface DataSourceOption { ds: GenericDataSource; } -export type DataSourceOptionType = EuiComboBoxOptionOption; - export interface DataSourceSelectableProps { dataSources: GenericDataSource[]; onDataSourceSelect: (dataSourceOption: DataSourceOption[]) => void; singleSelection?: boolean | EuiComboBoxSingleSelectionShape; - onFetchDataSetError: (error: Error) => void; + onGetDataSetError: (error: Error) => void; dataSourceOptionList: DataSourceGroup[]; selectedSources: DataSourceOption[]; setDataSourceOptionList: (dataSourceList: DataSourceGroup[]) => void; diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts index bff315d84ba7..2c8f393d7093 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.test.ts @@ -100,8 +100,8 @@ describe('DataSourceService', () => { const ds1 = new MockDataSource(mockConfig1); const ds2 = new MockDataSource(mockConfig2); service.registerMultipleDataSources([ds1, ds2]); - const filters = { names: ['test_datasource1'] }; - const retrievedDataSources = service.getDataSources(filters); + const filter = { names: ['test_datasource1'] }; + const retrievedDataSources = service.getDataSources(filter); expect(retrievedDataSources).toHaveProperty('test_datasource1'); expect(retrievedDataSources).not.toHaveProperty('test_datasource2'); }); diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index 3b0efcb4d660..ea7964cb8ec8 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -5,20 +5,20 @@ import { BehaviorSubject } from 'rxjs'; import { - IDataSourceFilters, - IDataSourceRegisterationResult, - DataSourceRegisterationError, + DataSourceRegistrationError, GenericDataSource, + IDataSourceFilter, + IDataSourceRegistrationResult, } from './types'; export class DataSourceService { private static dataSourceService: DataSourceService; // A record to store all registered data sources, using the data source name as the key. private dataSources: Record = {}; - private _dataSourcesSubject: BehaviorSubject>; + private dataSourcesSubject: BehaviorSubject>; private constructor() { - this._dataSourcesSubject = new BehaviorSubject(this.dataSources); + this.dataSourcesSubject = new BehaviorSubject(this.dataSources); } static getInstance(): DataSourceService { @@ -36,7 +36,7 @@ export class DataSourceService { */ async registerMultipleDataSources( datasources: GenericDataSource[] - ): Promise { + ): Promise { return Promise.all(datasources.map((ds) => this.registerDataSource(ds))); } @@ -46,46 +46,41 @@ export class DataSourceService { * * @param ds - The data source to be registered. * @returns A registration result indicating success or failure. - * @throws {DataSourceRegisterationError} Throws an error if a data source with the same name already exists. + * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. */ - async registerDataSource(ds: GenericDataSource): Promise { + async registerDataSource(ds: GenericDataSource): Promise { const dsName = ds.getName(); if (dsName in this.dataSources) { - throw new DataSourceRegisterationError( + throw new DataSourceRegistrationError( `Unable to register datasource ${dsName}, error: datasource name exists.` ); } else { - this.dataSources = { - ...this.dataSources, - [dsName]: ds, - }; - this._dataSourcesSubject.next(this.dataSources); - return { success: true, info: '' } as IDataSourceRegisterationResult; + this.dataSources[dsName] = ds; + this.dataSourcesSubject.next(this.dataSources); + return { success: true, info: '' } as IDataSourceRegistrationResult; } } public get dataSources$() { - return this._dataSourcesSubject.asObservable(); + return this.dataSourcesSubject.asObservable(); } /** * Retrieve the registered data sources based on provided filters. * If no filters are provided, all registered data sources are returned. * - * @param filters - An optional object with filter criteria (e.g., names of data sources). + * @param filter - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ - getDataSources(filters?: IDataSourceFilters): Record { - if (!Array.isArray(filters?.names) || filters.names.length === 0) return this.dataSources; + getDataSources(filter?: IDataSourceFilter): Record { + if (!filter || !Array.isArray(filter.names) || filter.names.length === 0) + return this.dataSources; - return filters.names.reduce>( - (filteredDataSources, dsName) => { - if (dsName in this.dataSources) { - filteredDataSources[dsName] = this.dataSources[dsName]; - } - return filteredDataSources; - }, - {} as Record - ); + return filter.names.reduce>((filteredDataSources, dsName) => { + if (dsName in this.dataSources) { + filteredDataSources[dsName] = this.dataSources[dsName]; + } + return filteredDataSources; + }, {} as Record); } } diff --git a/src/plugins/data/public/data_sources/datasource_services/index.ts b/src/plugins/data/public/data_sources/datasource_services/index.ts index eb9f40c9e2a7..14db278b47a5 100644 --- a/src/plugins/data/public/data_sources/datasource_services/index.ts +++ b/src/plugins/data/public/data_sources/datasource_services/index.ts @@ -5,9 +5,9 @@ export { DataSourceService } from './datasource_service'; export { - IDataSourceFilters, - IDataSourceRegisterationResult, - DataSourceRegisterationError, + IDataSourceFilter, + IDataSourceRegistrationResult, + DataSourceRegistrationError, DataSourceType, GenericDataSource, } from './types'; diff --git a/src/plugins/data/public/data_sources/datasource_services/types.ts b/src/plugins/data/public/data_sources/datasource_services/types.ts index 845329b61a61..022e9b9f178b 100644 --- a/src/plugins/data/public/data_sources/datasource_services/types.ts +++ b/src/plugins/data/public/data_sources/datasource_services/types.ts @@ -14,16 +14,16 @@ import { } from '../datasource'; import { DataSourceService } from './datasource_service'; -export interface IDataSourceFilters { +export interface IDataSourceFilter { names: string[]; } -export interface IDataSourceRegisterationResult { +export interface IDataSourceRegistrationResult { success: boolean; info: string; } -export class DataSourceRegisterationError extends Error { +export class DataSourceRegistrationError extends Error { success: boolean; info: string; constructor(message: string) { diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 16cebbcc78ec..3b559f9e6c63 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -510,11 +510,11 @@ export { DataSourceConfig, } from './data_sources/datasource'; export { - DataSourceRegisterationError, + DataSourceRegistrationError, DataSourceService, DataSourceType, - IDataSourceFilters, - IDataSourceRegisterationResult, + IDataSourceFilter, + IDataSourceRegistrationResult, } from './data_sources/datasource_services'; export { DataSourceSelectable, diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index 154040b31780..fa3235a08b6a 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,13 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -import React, { FC, useEffect, useState, useCallback } from 'react'; -import { EuiSplitPanel, EuiPageSideBar } from '@elastic/eui'; -import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; -import { DataExplorerServices } from '../../types'; -import { useTypedDispatch, useTypedSelector, setIndexPattern } from '../../utils/state_management'; +import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; +import { i18n } from '@osd/i18n'; +import React, { FC, useCallback, useEffect, useState } from 'react'; import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; +import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; +import { DataExplorerServices } from '../../types'; +import { setIndexPattern, useTypedDispatch, useTypedSelector } from '../../utils/state_management'; export const Sidebar: FC = ({ children }) => { const { indexPattern: indexPatternId } = useTypedSelector((state) => state.metadata); @@ -73,9 +74,14 @@ export const Sidebar: FC = ({ children }) => { dispatch(setIndexPattern(selectedDataSources[0].value)); }; - const handleDataSetFetchError = useCallback( + const handleGetDataSetError = useCallback( () => (error: Error) => { - toasts.addError(error, { title: `Data set fetching error: ${error}` }); + toasts.addError(error, { + title: + i18n.translate('dataExplorer.sidebar.failedToGetDataSetErrorDescription', { + defaultMessage: 'Failed to get data set: ', + }) + (error.message || error.name), + }); }, [toasts] ); @@ -90,7 +96,7 @@ export const Sidebar: FC = ({ children }) => { setDataSourceOptionList={setDataSourceOptionList} onDataSourceSelect={handleSourceSelection} selectedSources={selectedSources} - onFetchDataSetError={handleDataSetFetchError} + onGetDataSetError={handleGetDataSetError} /> From ff8f3efd5a82c89d23fa63bc98ef94f2acdbfac4 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 21:49:44 +0000 Subject: [PATCH 48/56] move functnions out of selectable component Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 645b96e9d1ee..07e6da0cea1a 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -34,14 +34,14 @@ const getDataSetWithSource = async (ds: GenericDataSource): Promise dataSources.map((ds) => getDataSetWithSource(ds)); -const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { +export const isIndexPatterns = (dataSet: DataSetType): dataSet is IndexPatternOption => { if (typeof dataSet === 'string') return false; return !!(dataSet.title && dataSet.id); }; // Get the option format for the combo box from the dataSource and dataSet. -const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { +export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetType) => { const optionContent = { type: dataSource.getType(), name: dataSource.getName(), From 0a1bfd54077efb0dc3f099097a23cc5c855d274f Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 21:58:43 +0000 Subject: [PATCH 49/56] add comments for dedup/options list updates Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 07e6da0cea1a..bd89102db7ff 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -76,8 +76,12 @@ const getSourceList = (allDataSets: ISourceDataSet[]) => { // check if add new datasource group or add to existing one if (existingGroup) { + // options deduplication const existingOptionIds = new Set(existingGroup.options.map((opt) => opt.label)); const nonDuplicateOptions = mappedOptions.filter((opt) => !existingOptionIds.has(opt.label)); + + // 'existingGroup' directly references an item in the finalList + // pushing options to 'existingGroup' updates the corresponding item in finalList existingGroup.options.push(...nonDuplicateOptions); } else { finalList.push({ From 21af094bd32549735487dc8845471a24b4562ce4 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 22:22:17 +0000 Subject: [PATCH 50/56] add experimental annotation Signed-off-by: Eric --- .../data/public/data_sources/datasource/datasource.ts | 5 +++-- src/plugins/data/public/data_sources/datasource/factory.ts | 3 +++ .../datasource_selector/datasource_selectable.tsx | 3 +++ .../data_sources/datasource_services/datasource_service.ts | 4 +++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index dda3f02255f4..4a168138746e 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -49,8 +49,8 @@ export abstract class DataSource< * all available tables for flint datasources, and get all index * patterns for OpenSearch data source * + * @experimental This API is experimental and might change in future releases. * @returns {SourceDataSet} Dataset associated with the data source. - * @experimental */ abstract getDataSet(dataSetParams?: DataSetParams): SourceDataSet; @@ -58,8 +58,8 @@ export abstract class DataSource< * Abstract method to run a query against the data source. * Implementing classes need to provide the specific implementation. * + * @experimental This API is experimental and might change in future releases. * @returns {DataSourceQueryResult} Result from querying the data source. - * @experimental */ abstract runQuery(queryParams: DataSourceQueryParams): DataSourceQueryResult; @@ -68,6 +68,7 @@ export abstract class DataSource< * Implementing classes should provide the specific logic to determine * the connection status, typically indicating success or failure. * + * @experimental This API is experimental and might change in future releases. * @returns {ConnectionStatus | Promise} Status of the connection test. * @experimental */ diff --git a/src/plugins/data/public/data_sources/datasource/factory.ts b/src/plugins/data/public/data_sources/datasource/factory.ts index 917c9c8ac168..f0b4e36cfb82 100644 --- a/src/plugins/data/public/data_sources/datasource/factory.ts +++ b/src/plugins/data/public/data_sources/datasource/factory.ts @@ -34,6 +34,7 @@ export class DataSourceFactory { /** * Returns the singleton instance of the DataSourceFactory. If it doesn't exist, it creates one. * + * @experimental This API is experimental and might change in future releases. * @returns {DataSourceFactory} The single instance of DataSourceFactory. */ static getInstance(): DataSourceFactory { @@ -47,6 +48,7 @@ export class DataSourceFactory { * Registers a new data source type with its associated class. * If the type has already been registered, an error is thrown. * + * @experimental This API is experimental and might change in future releases. * @param {string} type - The identifier for the data source type. * @param {DataSourceClass} dataSourceClass - The constructor of the data source class. * @throws {Error} Throws an error if the data source type has already been registered. @@ -62,6 +64,7 @@ export class DataSourceFactory { * Creates and returns an instance of the specified data source type with the given configuration. * If the type hasn't been registered, an error is thrown. * + * @experimental This API is experimental and might change in future releases. * @param {string} type - The identifier for the data source type. * @param {any} config - The configuration for the data source instance. * @returns {DataSourceType} An instance of the specified data source type. diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index bd89102db7ff..f42762fd5b28 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -93,6 +93,9 @@ const getSourceList = (allDataSets: ISourceDataSet[]) => { return finalList; }; +/** + * @experimental This component is experimental and might change in future releases. + */ export const DataSourceSelectable = ({ dataSources, // list of all available datasource connections. dataSourceOptionList, // combo box renderable option list derived from dataSources diff --git a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts index ea7964cb8ec8..9cf674585366 100644 --- a/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts +++ b/src/plugins/data/public/data_sources/datasource_services/datasource_service.ts @@ -31,6 +31,7 @@ export class DataSourceService { /** * Register multiple data sources at once. * + * @experimental This API is experimental and might change in future releases. * @param datasources - An array of data sources to be registered. * @returns An array of registration results, one for each data source. */ @@ -44,6 +45,7 @@ export class DataSourceService { * Register a single data source. * Throws an error if a data source with the same name is already registered. * + * @experimental This API is experimental and might change in future releases. * @param ds - The data source to be registered. * @returns A registration result indicating success or failure. * @throws {DataSourceRegistrationError} Throws an error if a data source with the same name already exists. @@ -68,7 +70,7 @@ export class DataSourceService { /** * Retrieve the registered data sources based on provided filters. * If no filters are provided, all registered data sources are returned. - * + * @experimental This API is experimental and might change in future releases. * @param filter - An optional object with filter criteria (e.g., names of data sources). * @returns A record of filtered data sources. */ From 24e677535c245fc1fc67618a7c2e43793db1a972 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 23:24:02 +0000 Subject: [PATCH 51/56] callback and experimental annotation on types Signed-off-by: Eric --- .../public/data_sources/datasource/types.ts | 4 + .../datasource_selectable.test.tsx | 143 +++++++++++++++++- .../datasource_selectable.tsx | 6 +- .../data_sources/datasource_selector/types.ts | 4 + .../data_sources/datasource_services/types.ts | 5 + 5 files changed, 160 insertions(+), 2 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/types.ts b/src/plugins/data/public/data_sources/datasource/types.ts index 8d7aa853fb07..bf77ef123a30 100644 --- a/src/plugins/data/public/data_sources/datasource/types.ts +++ b/src/plugins/data/public/data_sources/datasource/types.ts @@ -3,6 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +/** + * @experimental These interfaces are experimental and might change in future releases. + */ + import { IndexPatternsService } from '../../index_patterns'; import { DataSourceType } from '../datasource_services'; diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index bd424211325e..2a461811ad58 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -5,9 +5,57 @@ import React from 'react'; import { render, act } from '@testing-library/react'; -import { DataSourceSelectable } from './datasource_selectable'; +import { + DataSourceSelectable, + getSourceOptions, + fetchDataSetWithSource, + isIndexPatterns, +} from './datasource_selectable'; import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceOption } from './types'; +import { DataSource } from '../datasource/datasource'; +import { IndexPatternsService } from '../../index_patterns'; + +class MockDataSource extends DataSource { + private readonly indexPattern; + + constructor({ + name, + type, + metadata, + indexPattern, + }: { + name: string; + type: string; + metadata: any; + indexPattern: IndexPatternsService; + }) { + super(name, type, metadata); + this.indexPattern = indexPattern; + } + + async getDataSet(dataSetParams?: any) { + await this.indexPattern.ensureDefaultIndexPattern(); + return await this.indexPattern.getCache(); + } + + async testConnection(): Promise { + return true; + } + + async runQuery(queryParams: any) { + return undefined; + } +} + +const mockIndexPattern = {} as IndexPatternsService; + +const mockConfig = { + name: 'test_datasource1', + type: 'mock1', + metadata: null, + indexPattern: mockIndexPattern, +}; describe('DataSourceSelectable', () => { let dataSourcesMock: GenericDataSource[]; @@ -16,6 +64,25 @@ describe('DataSourceSelectable', () => { let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); + const mockDataSources = { + getType: jest.fn().mockReturnValue('type1'), + getName: jest.fn().mockReturnValue('source1'), + }; + + it('should fetch data set and return structured data', async () => { + const mockDs = { + getDataSet: jest.fn().mockResolvedValue(['dataItem1', 'dataItem2']), + getType: jest.fn(), + getName: jest.fn(), + }; + + const result = await fetchDataSetWithSource(mockDs as any); + expect(result).toEqual({ + ds: mockDs, + data_sets: ['dataItem1', 'dataItem2'], + }); + }); + const ds = new MockDataSource(mockConfig); beforeEach(() => { dataSourcesMock = [ @@ -81,4 +148,78 @@ describe('DataSourceSelectable', () => { expect(onFetchDataSetErrorMock).toHaveBeenCalledWith(new Error('Fetch error')); }); + + it('returns the matched option when found', () => { + const allDataSets = [ + { + ds: mockDataSources, + data_sets: [ + { title: 'index1', id: 'id1' }, + { title: 'index', id: 'id2' }, + ], + }, + { + ds: mockDataSources, + data_sets: [ + { title: 'index1', id: 'id1' }, + { title: 'index', id: 'id2' }, + ], + }, + ]; + + const result = getSourceOptions(ds, allDataSets); + const expected = [ + { + label: 'Type Display Name for type1 or Default Group', // Assuming your DATASOURCE_TYPE_DISPLAY_NAME_MAP maps type1 to this + options: [ + { label: 'index1', value: 'id1', type: 'type1', name: 'source1', ds: mockDataSources }, + { label: 'index1', value: 'id2', type: 'type1', name: 'source1', ds: mockDataSources }, + ], + }, + ]; + + expect(result).toEqual(expected); + }); + + it('handles nested data source groups correctly', () => { + const allDataSets = [ + // ... Your mock data that tests the nested group handling ... + ]; + + const result = getSourceOptions(allDataSets); + + // Write your expected output here + const expected = [ + // ... Your expected data source groups ... + ]; + + expect(result).toEqual(expected); + }); + + it('returns undefined in option value when no match is found', () => { + const allDataSets = [ + { + ds: mockDataSources, + data_sets: ['data1'], // This should not match because it's a string + }, + ]; + + const result = getSourceOptions(allDataSets); + const expected = [ + { + label: 'Type Display Name for type1 or Default Group', + options: [ + { + label: 'source1', + value: 'source1', + type: 'type1', + name: 'source1', + ds: mockDataSources, + }, + ], + }, + ]; + + expect(result).toEqual(expected); + }); }); diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index f42762fd5b28..65c8b7b5f8a7 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -9,6 +9,7 @@ import React, { useEffect } from 'react'; import { ISourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; +import { useCallback } from 'react'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; @@ -114,7 +115,10 @@ export const DataSourceSelectable = ({ .catch((e) => onGetDataSetError(e)); }, [dataSources, setDataSourceOptionList, onGetDataSetError]); - const handleSourceChange = (selectedOptions: any) => onDataSourceSelect(selectedOptions); + const handleSourceChange = useCallback( + (selectedOptions: any) => onDataSourceSelect(selectedOptions), + [onDataSourceSelect] + ); return ( Date: Thu, 5 Oct 2023 23:41:30 +0000 Subject: [PATCH 52/56] handleSourceSelection callback Signed-off-by: Eric --- .../public/components/sidebar/index.tsx | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index fa3235a08b6a..acf4124da1f6 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -58,21 +58,24 @@ export const Sidebar: FC = ({ children }) => { } }, [indexPatternId, activeDataSources, dataSourceOptionList]); - const handleSourceSelection = (selectedDataSources: DataSourceOption[]) => { - if (selectedDataSources.length === 0) { + const handleSourceSelection = useCallback( + (selectedDataSources: DataSourceOption[]) => { + if (selectedDataSources.length === 0) { + setSelectedSources(selectedDataSources); + return; + } + // Temporary redirection solution for 2.11, where clicking non-index-pattern datasource + // will redirect user to Observability event explorer + if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { + return application.navigateToUrl( + `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` + ); + } setSelectedSources(selectedDataSources); - return; - } - // Temporary redirection solution for 2.11, where clicking non-index-pattern datasource - // will redirect user to Observability event explorer - if (selectedDataSources[0]?.ds?.getType() !== 'DEFAULT_INDEX_PATTERNS') { - return application.navigateToUrl( - `../observability-logs#/explorer?datasourceName=${selectedDataSources[0].label}&datasourceType=${selectedDataSources[0].type}` - ); - } - setSelectedSources(selectedDataSources); - dispatch(setIndexPattern(selectedDataSources[0].value)); - }; + dispatch(setIndexPattern(selectedDataSources[0].value)); + }, + [application, dispatch] + ); const handleGetDataSetError = useCallback( () => (error: Error) => { From 81ff8661f0f781c31d9fda506303b76ddde26756 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 23:49:49 +0000 Subject: [PATCH 53/56] datasource annotation and import adjustment Signed-off-by: Eric --- src/plugins/data/public/data_sources/datasource/datasource.ts | 3 +++ .../data_sources/datasource_selector/datasource_selectable.tsx | 3 +-- src/plugins/data_explorer/public/components/sidebar/index.tsx | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource/datasource.ts b/src/plugins/data/public/data_sources/datasource/datasource.ts index 4a168138746e..a2159c562064 100644 --- a/src/plugins/data/public/data_sources/datasource/datasource.ts +++ b/src/plugins/data/public/data_sources/datasource/datasource.ts @@ -15,6 +15,9 @@ import { ConnectionStatus } from './types'; +/** + * @experimental this class is experimental and might change in future releases. + */ export abstract class DataSource< DataSourceMetaData, DataSetParams, diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 65c8b7b5f8a7..8823117b92a2 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -3,13 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ +import React, { useEffect, useCallback } from 'react'; import { EuiComboBox } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import React, { useEffect } from 'react'; import { ISourceDataSet, IndexPatternOption } from '../datasource'; import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceSelectableProps } from './types'; -import { useCallback } from 'react'; type DataSourceTypeKey = 'DEFAULT_INDEX_PATTERNS' | 's3glue' | 'spark'; diff --git a/src/plugins/data_explorer/public/components/sidebar/index.tsx b/src/plugins/data_explorer/public/components/sidebar/index.tsx index acf4124da1f6..14bd35656385 100644 --- a/src/plugins/data_explorer/public/components/sidebar/index.tsx +++ b/src/plugins/data_explorer/public/components/sidebar/index.tsx @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +import React, { FC, useCallback, useEffect, useState } from 'react'; import { EuiPageSideBar, EuiSplitPanel } from '@elastic/eui'; import { i18n } from '@osd/i18n'; -import React, { FC, useCallback, useEffect, useState } from 'react'; import { DataSourceGroup, DataSourceSelectable, DataSourceType } from '../../../../data/public'; import { DataSourceOption } from '../../../../data/public/'; import { useOpenSearchDashboards } from '../../../../opensearch_dashboards_react/public'; From 30fc3d6e135212e983810206a39e1f79e7a68fc7 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 5 Oct 2023 23:55:13 +0000 Subject: [PATCH 54/56] remove pill effect Signed-off-by: Eric --- .../data_sources/datasource_selector/datasource_selectable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index 8823117b92a2..a9c6f681b01f 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -103,7 +103,7 @@ export const DataSourceSelectable = ({ onDataSourceSelect, setDataSourceOptionList, onGetDataSetError, // onGetDataSetError, Callback for handling get data set errors. Ensure it's memoized. - singleSelection = true, + singleSelection = { asPlainText: true }, }: DataSourceSelectableProps) => { // This effect gets data sets and prepares the datasource list for UI rendering. useEffect(() => { From 5ab664beccfca95f52bd00b2c8014586c69a110b Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 6 Oct 2023 00:00:39 +0000 Subject: [PATCH 55/56] minor changes addressing latest comments Signed-off-by: Eric --- .../datasource_selector/datasource_selectable.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx index a9c6f681b01f..77b44c206d6d 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.tsx @@ -48,11 +48,10 @@ export const getSourceOptions = (dataSource: DataSourceType, dataSet: DataSetTyp ds: dataSource, }; if (isIndexPatterns(dataSet)) { - const ip = dataSet as IndexPatternOption; return { ...optionContent, - label: ip.title, - value: ip.id, + label: dataSet.title, + value: dataSet.id, }; } return { From bea8e75b817868811433c67921a8916be0a7e3c0 Mon Sep 17 00:00:00 2001 From: Eric Date: Fri, 6 Oct 2023 00:59:24 +0000 Subject: [PATCH 56/56] remove unused tests Signed-off-by: Eric --- .../datasource_selectable.test.tsx | 143 +----------------- 1 file changed, 1 insertion(+), 142 deletions(-) diff --git a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx index 2a461811ad58..bd424211325e 100644 --- a/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx +++ b/src/plugins/data/public/data_sources/datasource_selector/datasource_selectable.test.tsx @@ -5,57 +5,9 @@ import React from 'react'; import { render, act } from '@testing-library/react'; -import { - DataSourceSelectable, - getSourceOptions, - fetchDataSetWithSource, - isIndexPatterns, -} from './datasource_selectable'; +import { DataSourceSelectable } from './datasource_selectable'; import { DataSourceType, GenericDataSource } from '../datasource_services'; import { DataSourceGroup, DataSourceOption } from './types'; -import { DataSource } from '../datasource/datasource'; -import { IndexPatternsService } from '../../index_patterns'; - -class MockDataSource extends DataSource { - private readonly indexPattern; - - constructor({ - name, - type, - metadata, - indexPattern, - }: { - name: string; - type: string; - metadata: any; - indexPattern: IndexPatternsService; - }) { - super(name, type, metadata); - this.indexPattern = indexPattern; - } - - async getDataSet(dataSetParams?: any) { - await this.indexPattern.ensureDefaultIndexPattern(); - return await this.indexPattern.getCache(); - } - - async testConnection(): Promise { - return true; - } - - async runQuery(queryParams: any) { - return undefined; - } -} - -const mockIndexPattern = {} as IndexPatternsService; - -const mockConfig = { - name: 'test_datasource1', - type: 'mock1', - metadata: null, - indexPattern: mockIndexPattern, -}; describe('DataSourceSelectable', () => { let dataSourcesMock: GenericDataSource[]; @@ -64,25 +16,6 @@ describe('DataSourceSelectable', () => { let setSelectedSourcesMock: (sources: DataSourceOption[]) => void = jest.fn(); let setDataSourceOptionListMock: (sources: DataSourceGroup[]) => void = jest.fn(); let onFetchDataSetErrorMock: (error: Error) => void = jest.fn(); - const mockDataSources = { - getType: jest.fn().mockReturnValue('type1'), - getName: jest.fn().mockReturnValue('source1'), - }; - - it('should fetch data set and return structured data', async () => { - const mockDs = { - getDataSet: jest.fn().mockResolvedValue(['dataItem1', 'dataItem2']), - getType: jest.fn(), - getName: jest.fn(), - }; - - const result = await fetchDataSetWithSource(mockDs as any); - expect(result).toEqual({ - ds: mockDs, - data_sets: ['dataItem1', 'dataItem2'], - }); - }); - const ds = new MockDataSource(mockConfig); beforeEach(() => { dataSourcesMock = [ @@ -148,78 +81,4 @@ describe('DataSourceSelectable', () => { expect(onFetchDataSetErrorMock).toHaveBeenCalledWith(new Error('Fetch error')); }); - - it('returns the matched option when found', () => { - const allDataSets = [ - { - ds: mockDataSources, - data_sets: [ - { title: 'index1', id: 'id1' }, - { title: 'index', id: 'id2' }, - ], - }, - { - ds: mockDataSources, - data_sets: [ - { title: 'index1', id: 'id1' }, - { title: 'index', id: 'id2' }, - ], - }, - ]; - - const result = getSourceOptions(ds, allDataSets); - const expected = [ - { - label: 'Type Display Name for type1 or Default Group', // Assuming your DATASOURCE_TYPE_DISPLAY_NAME_MAP maps type1 to this - options: [ - { label: 'index1', value: 'id1', type: 'type1', name: 'source1', ds: mockDataSources }, - { label: 'index1', value: 'id2', type: 'type1', name: 'source1', ds: mockDataSources }, - ], - }, - ]; - - expect(result).toEqual(expected); - }); - - it('handles nested data source groups correctly', () => { - const allDataSets = [ - // ... Your mock data that tests the nested group handling ... - ]; - - const result = getSourceOptions(allDataSets); - - // Write your expected output here - const expected = [ - // ... Your expected data source groups ... - ]; - - expect(result).toEqual(expected); - }); - - it('returns undefined in option value when no match is found', () => { - const allDataSets = [ - { - ds: mockDataSources, - data_sets: ['data1'], // This should not match because it's a string - }, - ]; - - const result = getSourceOptions(allDataSets); - const expected = [ - { - label: 'Type Display Name for type1 or Default Group', - options: [ - { - label: 'source1', - value: 'source1', - type: 'type1', - name: 'source1', - ds: mockDataSources, - }, - ], - }, - ]; - - expect(result).toEqual(expected); - }); });