Update import api to have data source id to allow import saved objects from uploading files to have data source

CHANGELOG.md

@@ -346,6 +346,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Dashboard De-Angular] Add more unit tests for utils folder ([#4641](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4641))
 - [Dashboard De-Angular] Add unit tests for dashboard_listing and dashboard_top_nav ([#4640](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4640))
 - Optimize `augment-vis` saved obj searching by adding arg to saved obj client ([#4595](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4595))
+- Change SavedObjects' Import API to allow selecting a data source when uploading files ([#5777](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/5777))
 
 ### 🐛 Bug Fixes
 

config/opensearch_dashboards.yml

@@ -274,4 +274,4 @@
 # opensearchDashboards.survey.url: "https://survey.opensearch.org"
 
 # Set the value of this setting to true to enable plugin augmentation on Dashboard
-# vis_augmenter.pluginAugmentationEnabled: true
+# vis_augmenter.pluginAugmentationEnabled: true

src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts

@@ -0,0 +1,150 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { mockUuidv4 } from './__mocks__';
+import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public';
+import { SavedObject } from '../types';
+import { SavedObjectsErrorHelpers } from '..';
+import {
+  checkConflictsForDataSource,
+  ConflictsForDataSourceParams,
+} from './check_conflict_for_data_source';
+
+type SavedObjectType = SavedObject<{ title?: string }>;
+
+/**
+ * Function to create a realistic-looking import object given a type and ID
+ */
+const createObject = (type: string, id: string): SavedObjectType => ({
+  type,
+  id,
+  attributes: { title: 'some-title' },
+  references: (Symbol() as unknown) as SavedObjectReference[],
+});
+
+const getResultMock = {
+  conflict: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createConflictError(type, id).output.payload;
+    return { type, id, error };
+  },
+  unresolvableConflict: (type: string, id: string) => {
+    const conflictMock = getResultMock.conflict(type, id);
+    const metadata = { isNotOverwritable: true };
+    return { ...conflictMock, error: { ...conflictMock.error, metadata } };
+  },
+  invalidType: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createUnsupportedTypeError(type).output.payload;
+    return { type, id, error };
+  },
+};
+
+/**
+ * Create a variety of different objects to exercise different import / result scenarios
+ */
+const dataSourceObj = createObject('data-source', 'data-source-id-1'); // -> data-source type, no need to add in the filteredObjects
+const dataSourceObj1 = createObject('type-1', 'ds_id-1'); // -> object with data source id
+const dataSourceObj2 = createObject('type-2', 'ds_id-2'); // -> object with data source id
+const objectsWithDataSource = [dataSourceObj, dataSourceObj1, dataSourceObj2];
+const dataSourceObj1Error = getResultMock.conflict(dataSourceObj1.type, dataSourceObj1.id);
+
+describe('#checkConflictsForDataSource', () => {
+  const setupParams = (partial: {
+    objects: SavedObjectType[];
+    ignoreRegularConflicts?: boolean;
+    retries?: SavedObjectsImportRetry[];
+    createNewCopies?: boolean;
+    dataSourceId?: string;
+  }): ConflictsForDataSourceParams => {
+    return { ...partial };
+  };
+
+  beforeEach(() => {
+    mockUuidv4.mockReset();
+    mockUuidv4.mockReturnValueOnce(`new-object-id`);
+  });
+
+  it('exits early if there are no objects to check', async () => {
+    const params = setupParams({ objects: [] });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('return obj if it is not data source obj and there is no conflict of the data source id', async () => {
+    const params = setupParams({ objects: objectsWithDataSource, dataSourceId: 'ds' });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [dataSourceObj1, dataSourceObj2],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('can resolve the data source id conflict when the ds it not match when ignoreRegularConflicts=true', async () => {
+    const params = setupParams({
+      objects: objectsWithDataSource,
+      ignoreRegularConflicts: true,
+      dataSourceId: 'currentDsId',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+
+    expect(checkConflictsForDataSourceResult).toEqual(
+      expect.objectContaining({
+        filteredObjects: [
+          {
+            ...dataSourceObj1,
+            id: 'currentDsId_id-1',
+          },
+          {
+            ...dataSourceObj2,
+            id: 'currentDsId_id-2',
+          },
+        ],
+        errors: [],
+        importIdMap: new Map([
+          [
+            `${dataSourceObj1.type}:${dataSourceObj1.id}`,
+            { id: 'currentDsId_id-1', omitOriginId: true },
+          ],
+          [
+            `${dataSourceObj2.type}:${dataSourceObj2.id}`,
+            { id: 'currentDsId_id-2', omitOriginId: true },
+          ],
+        ]),
+        pendingOverwrites: new Set([
+          `${dataSourceObj1.type}:${dataSourceObj1.id}`,
+          `${dataSourceObj2.type}:${dataSourceObj2.id}`,
+        ]),
+      })
+    );
+  });
+
+  it('can push error when do not override with data source conflict', async () => {
+    const params = setupParams({
+      objects: [dataSourceObj1],
+      ignoreRegularConflicts: false,
+      dataSourceId: 'currentDs',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [
+        {
+          ...dataSourceObj1Error,
+          title: dataSourceObj1.attributes.title,
+          meta: { title: dataSourceObj1.attributes.title },
+          error: { type: 'conflict' },
+        },
+      ],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+});

src/core/server/saved_objects/import/check_conflict_for_data_source.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { SavedObject, SavedObjectsImportError, SavedObjectsImportRetry } from '../types';
+
+export interface ConflictsForDataSourceParams {
+  objects: Array<SavedObject<{ title?: string }>>;
+  ignoreRegularConflicts?: boolean;
+  retries?: SavedObjectsImportRetry[];
+  dataSourceId?: string;
+}
+
+interface ImportIdMapEntry {
+  id?: string;
+  omitOriginId?: boolean;
+}
+
+/**
+ * function to check the conflict when multiple data sources are enabled.
+ * the purpose of this function is to check the conflict of the imported saved objects and data source
+ * @param objects, this the array of saved objects to be verified whether contains the data source conflict
+ * @param ignoreRegularConflicts whether to override
+ * @param retries import operations list
+ * @param dataSourceId the id to identify the data source
+ * @returns {filteredObjects, errors, importIdMap, pendingOverwrites }
+ */
+export async function checkConflictsForDataSource({
+  objects,
+  ignoreRegularConflicts,
+  retries = [],
+  dataSourceId,
+}: ConflictsForDataSourceParams) {
+  const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
+  const errors: SavedObjectsImportError[] = [];
+  const importIdMap = new Map<string, ImportIdMapEntry>();
+  const pendingOverwrites = new Set<string>();
+
+  // exit early if there are no objects to check
+  if (objects.length === 0) {
+    return { filteredObjects, errors, importIdMap, pendingOverwrites };
+  }
+  const retryMap = retries.reduce(
+    (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur),
+    new Map<string, SavedObjectsImportRetry>()
+  );
+  objects.forEach((object) => {
+    const {
+      type,
+      id,
+      attributes: { title },
+    } = object;
+    const { destinationId } = retryMap.get(`${type}:${id}`) || {};
+
+    if (object.type !== 'data-source') {
+      const parts = id.split('_'); // this is the array to host the split results of the id
+      const previoudDataSourceId = parts.length > 1 ? parts[0] : undefined;
+      const rawId = previoudDataSourceId ? parts[1] : parts[0];
+
+      /**
+       * for import saved object from osd exported
+       * when the imported saved objects with the different dataSourceId comparing to the current dataSourceId
+       */
+
+      if (
+        previoudDataSourceId &&
+        previoudDataSourceId !== dataSourceId &&
+        !ignoreRegularConflicts
+      ) {
+        const error = { type: 'conflict' as 'conflict', ...(destinationId && { destinationId }) };
+        errors.push({ type, id, title, meta: { title }, error });
+      } else if (previoudDataSourceId && previoudDataSourceId === dataSourceId) {
+        filteredObjects.push(object);
+      } else {
+        const omitOriginId = ignoreRegularConflicts;
+        importIdMap.set(`${type}:${id}`, { id: `${dataSourceId}_${rawId}`, omitOriginId });
+        pendingOverwrites.add(`${type}:${id}`);
+        filteredObjects.push({ ...object, id: `${dataSourceId}_${rawId}` });
+      }
+    }
+  });
+
+  return { filteredObjects, errors, importIdMap, pendingOverwrites };
+}

src/core/server/saved_objects/import/check_origin_conflicts.ts

@@ -45,6 +45,7 @@ interface CheckOriginConflictsParams {
   namespace?: string;
   ignoreRegularConflicts?: boolean;
   importIdMap: Map<string, unknown>;
+  dataSourceId?: string;
 }
 
 type CheckOriginConflictParams = Omit<CheckOriginConflictsParams, 'objects'> & {

src/core/server/saved_objects/import/collect_saved_objects.ts

@@ -46,13 +46,15 @@ interface CollectSavedObjectsOptions {
   objectLimit: number;
   filter?: (obj: SavedObject) => boolean;
   supportedTypes: string[];
+  dataSourceId?: string;
 }
 
 export async function collectSavedObjects({
   readStream,
   objectLimit,
   filter,
   supportedTypes,
+  dataSourceId,
 }: CollectSavedObjectsOptions) {
   const errors: SavedObjectsImportError[] = [];
   const entries: Array<{ type: string; id: string }> = [];