diff --git a/changelogs/fragments/6443.yml b/changelogs/fragments/6443.yml new file mode 100644 index 000000000000..2de7191e0d09 --- /dev/null +++ b/changelogs/fragments/6443.yml @@ -0,0 +1,2 @@ +feat: +- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443)) \ No newline at end of file diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml index 9208e8e39729..3ebcfe77cd16 100644 --- a/config/opensearch_dashboards.yml +++ b/config/opensearch_dashboards.yml @@ -322,6 +322,12 @@ # Set the value to true to enable workspace feature # workspace.enabled: false +# Optional settings to specify saved object types to be deleted during migration. +# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications. +# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution. +# migrations.delete.enabled: false +# migrations.delete.types: [] + # Set the value to true to enable Ui Metric Collectors in Usage Collector # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false # usageCollection.uiMetric.enabled: false diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts index 296ca5f29f1e..5974d517d5f3 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.test.ts @@ -435,6 +435,228 @@ describe('IndexMigrator', () => { }); }); + test('deletes saved objects by type if configured', async () => { + const { client } = testOpts; + + const deleteType = 'delete_type'; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return true; + } + if (path === 'migrations.delete.types') { + return [deleteType]; + } + }); + testOpts.opensearchDashboardsRawConfig = rawConfig; + + testOpts.mappingProperties = { foo: { type: 'text' } as any }; + + withIndex(client, { + index: { + '.kibana_1': { + aliases: {}, + mappings: { + properties: { + delete_type: { properties: { type: deleteType } }, + }, + }, + }, + }, + }); + + await new IndexMigrator(testOpts).migrate(); + + expect(client.indices.create).toHaveBeenCalledWith({ + body: { + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: { + foo: '625b32086eb1d1203564cf85062dd22e', + migrationVersion: '4a1746014a75ade3a714e1db5763276f', + namespace: '2f4316de49999235636386fe51dc06c1', + namespaces: '2f4316de49999235636386fe51dc06c1', + originId: '2f4316de49999235636386fe51dc06c1', + references: '7997cf5a56cc02bdc9c93361bde732b0', + type: '2f4316de49999235636386fe51dc06c1', + updated_at: '00da57df13e94e9d98437d13ace4bfe0', + }, + }, + properties: { + foo: { type: 'text' }, + migrationVersion: { dynamic: 'true', type: 'object' }, + namespace: { type: 'keyword' }, + namespaces: { type: 'keyword' }, + originId: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + references: { + type: 'nested', + properties: { + name: { type: 'keyword' }, + type: { type: 'keyword' }, + id: { type: 'keyword' }, + }, + }, + }, + }, + settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, + }, + index: '.kibana_2', + }); + }); + + test('retains saved objects by type if delete is not enabled', async () => { + const { client } = testOpts; + + const deleteType = 'delete_type'; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return false; + } + if (path === 'migrations.delete.types') { + return [deleteType]; + } + }); + testOpts.opensearchDashboardsRawConfig = rawConfig; + + testOpts.mappingProperties = { foo: { type: 'text' } as any }; + + withIndex(client, { + index: { + '.kibana_1': { + aliases: {}, + mappings: { + properties: { + delete_type: { properties: { type: deleteType } }, + }, + }, + }, + }, + }); + + await new IndexMigrator(testOpts).migrate(); + + expect(client.indices.create).toHaveBeenCalledWith({ + body: { + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: { + foo: '625b32086eb1d1203564cf85062dd22e', + migrationVersion: '4a1746014a75ade3a714e1db5763276f', + namespace: '2f4316de49999235636386fe51dc06c1', + namespaces: '2f4316de49999235636386fe51dc06c1', + originId: '2f4316de49999235636386fe51dc06c1', + references: '7997cf5a56cc02bdc9c93361bde732b0', + type: '2f4316de49999235636386fe51dc06c1', + updated_at: '00da57df13e94e9d98437d13ace4bfe0', + }, + }, + properties: { + delete_type: { dynamic: false, properties: {} }, + foo: { type: 'text' }, + migrationVersion: { dynamic: 'true', type: 'object' }, + namespace: { type: 'keyword' }, + namespaces: { type: 'keyword' }, + originId: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + references: { + type: 'nested', + properties: { + name: { type: 'keyword' }, + type: { type: 'keyword' }, + id: { type: 'keyword' }, + }, + }, + }, + }, + settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, + }, + index: '.kibana_2', + }); + }); + + test('retains saved objects by type if delete types does not exist', async () => { + const { client } = testOpts; + + const deleteType = 'delete_type'; + const retainType = 'retain_type'; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return true; + } + if (path === 'migrations.delete.types') { + return [deleteType]; + } + }); + testOpts.opensearchDashboardsRawConfig = rawConfig; + + testOpts.mappingProperties = { foo: { type: 'text' } as any }; + + withIndex(client, { + index: { + '.kibana_1': { + aliases: {}, + mappings: { + properties: { + retain_type: { properties: { type: retainType } }, + }, + }, + }, + }, + }); + + await new IndexMigrator(testOpts).migrate(); + + expect(client.indices.create).toHaveBeenCalledWith({ + body: { + mappings: { + dynamic: 'strict', + _meta: { + migrationMappingPropertyHashes: { + foo: '625b32086eb1d1203564cf85062dd22e', + migrationVersion: '4a1746014a75ade3a714e1db5763276f', + namespace: '2f4316de49999235636386fe51dc06c1', + namespaces: '2f4316de49999235636386fe51dc06c1', + originId: '2f4316de49999235636386fe51dc06c1', + references: '7997cf5a56cc02bdc9c93361bde732b0', + type: '2f4316de49999235636386fe51dc06c1', + updated_at: '00da57df13e94e9d98437d13ace4bfe0', + }, + }, + properties: { + retain_type: { dynamic: false, properties: {} }, + foo: { type: 'text' }, + migrationVersion: { dynamic: 'true', type: 'object' }, + namespace: { type: 'keyword' }, + namespaces: { type: 'keyword' }, + originId: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + references: { + type: 'nested', + properties: { + name: { type: 'keyword' }, + type: { type: 'keyword' }, + id: { type: 'keyword' }, + }, + }, + }, + }, + settings: { number_of_shards: 1, auto_expand_replicas: '0-1' }, + }, + index: '.kibana_2', + }); + }); + test('points the alias at the dest index', async () => { const { client } = testOpts; diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts index 1a616f8a2c7d..20784d6db8f6 100644 --- a/src/core/server/saved_objects/migrations/core/index_migrator.ts +++ b/src/core/server/saved_objects/migrations/core/index_migrator.ts @@ -28,6 +28,7 @@ * under the License. */ +import { DeleteByQueryRequest } from '@opensearch-project/opensearch/api/types'; import { diffMappings } from './build_active_mappings'; import * as Index from './opensearch_index'; import { migrateRawDocs } from './migrate_raw_docs'; @@ -123,6 +124,7 @@ async function migrateIndex(context: Context): Promise { const { client, alias, source, dest, log } = context; await deleteIndexTemplates(context); + await deleteSavedObjectsByType(context); log.info(`Creating index ${dest.indexName}.`); @@ -171,6 +173,33 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! }))); } +/** + * Delete saved objects by type. If migrations.delete.types is specified, + * any saved objects that matches that type will be deleted. + */ +async function deleteSavedObjectsByType(context: Context) { + const { client, source, log, typesToDelete } = context; + if (!source.exists || !typesToDelete || typesToDelete.length === 0) { + return; + } + + log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`); + const params = { + index: source.indexName, + body: { + query: { + bool: { + should: [...typesToDelete.map((type) => ({ term: { type } }))], + }, + }, + }, + conflicts: 'proceed', + refresh: true, + } as DeleteByQueryRequest; + log.debug(`Delete by query params: ${JSON.stringify(params)}`); + return client.deleteByQuery(params); +} + /** * Moves all docs from sourceIndex to destIndex, migrating each as necessary. * This moves documents from the concrete index, rather than the alias, to prevent diff --git a/src/core/server/saved_objects/migrations/core/migration_context.test.ts b/src/core/server/saved_objects/migrations/core/migration_context.test.ts index 71db15842cd3..c30e6910cbf4 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.test.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.test.ts @@ -28,7 +28,8 @@ * under the License. */ -import { disableUnknownTypeMappingFields } from './migration_context'; +import { disableUnknownTypeMappingFields, deleteTypeMappingsFields } from './migration_context'; +import { configMock } from '../../../config/mocks'; describe('disableUnknownTypeMappingFields', () => { const sourceMappings = { @@ -97,3 +98,87 @@ describe('disableUnknownTypeMappingFields', () => { }); }); }); + +describe('deleteTypeMappingsFields', () => { + it('should delete specified type mappings fields', () => { + const targetMappings = { + properties: { + type1: { type: 'text' }, + type2: { type: 'keyword' }, + type3: { type: 'boolean' }, + }, + } as const; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return true; + } + if (path === 'migrations.delete.types') { + return ['type1', 'type3']; + } + }); + + const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig); + + expect(updatedMappings.properties).toEqual({ + type2: { type: 'keyword' }, + }); + }); + + it('should not delete any type mappings fields if delete is not enabled', () => { + const targetMappings = { + properties: { + type1: { type: 'text' }, + type2: { type: 'keyword' }, + type3: { type: 'boolean' }, + }, + } as const; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return false; + } + if (path === 'migrations.delete.types') { + return ['type1', 'type3']; + } + }); + + const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig); + + expect(updatedMappings.properties).toEqual({ + type1: { type: 'text' }, + type2: { type: 'keyword' }, + type3: { type: 'boolean' }, + }); + }); + + it('should not delete any type mappings fields if delete types are not specified', () => { + const targetMappings = { + properties: { + type1: { type: 'text' }, + type2: { type: 'keyword' }, + type3: { type: 'boolean' }, + }, + } as const; + + const rawConfig = configMock.create(); + rawConfig.get.mockImplementation((path) => { + if (path === 'migrations.delete.enabled') { + return true; + } + if (path === 'migrations.delete.types') { + return []; + } + }); + + const updatedMappings = deleteTypeMappingsFields(targetMappings, rawConfig); + + expect(updatedMappings.properties).toEqual({ + type1: { type: 'text' }, + type2: { type: 'keyword' }, + type3: { type: 'boolean' }, + }); + }); +}); diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts index 91114701d95f..987115c2ce08 100644 --- a/src/core/server/saved_objects/migrations/core/migration_context.ts +++ b/src/core/server/saved_objects/migrations/core/migration_context.ts @@ -66,6 +66,11 @@ export interface MigrationOpts { * prior to running migrations. For example: 'opensearch_dashboards_index_template*' */ obsoleteIndexTemplatePattern?: string; + /** + * If specified, types matching the specified list will be removed prior to + * running migrations. Useful for removing types that are not supported. + */ + typesToDelete?: string[]; opensearchDashboardsRawConfig?: Config; } @@ -84,6 +89,7 @@ export interface Context { scrollDuration: string; serializer: SavedObjectsSerializer; obsoleteIndexTemplatePattern?: string; + typesToDelete?: string[]; convertToAliasScript?: string; } @@ -114,6 +120,7 @@ export async function migrationContext(opts: MigrationOpts): Promise { scrollDuration: opts.scrollDuration, serializer: opts.serializer, obsoleteIndexTemplatePattern: opts.obsoleteIndexTemplatePattern, + typesToDelete: opts.typesToDelete, convertToAliasScript: opts.convertToAliasScript, }; } @@ -135,9 +142,12 @@ function createDestContext( typeMappingDefinitions: SavedObjectsTypeMappingDefinitions, opensearchDashboardsRawConfig?: Config ): Index.FullIndexInfo { - const targetMappings = disableUnknownTypeMappingFields( - buildActiveMappings(typeMappingDefinitions, opensearchDashboardsRawConfig), - source.mappings + const targetMappings = deleteTypeMappingsFields( + disableUnknownTypeMappingFields( + buildActiveMappings(typeMappingDefinitions, opensearchDashboardsRawConfig), + source.mappings + ), + opensearchDashboardsRawConfig ); return { @@ -162,7 +172,7 @@ function createDestContext( * type's mappings are set to `dynamic: false`. * * (Since we're using the source index mappings instead of looking at actual - * document types in the inedx, we potentially add more "unknown types" than + * document types in the index, we potentially add more "unknown types" than * what would be necessary to support migrating all the data over to the * target index.) * @@ -199,6 +209,43 @@ export function disableUnknownTypeMappingFields( }; } +/** + * This function is used to modify the target mappings object by deleting specified type mappings fields. + * + * The function operates under the following conditions: + * - It checks if the 'migrations.delete.enabled' configuration is set to true. + * - If true, it retrieves the 'migrations.delete.types' configuration + * - For each type, it deletes the corresponding property from the target mappings object. + * + * The purpose of this function is to allow for dynamic modification of the target mappings object + * based on the application's configuration. This can be useful in scenarios where certain type + * mappings are no longer needed and should be removed from the target mappings. + * + * @param {Object} targetMappings - The target mappings object to be modified. + * @param {Object} opensearchDashboardsRawConfig - The application's configuration object. + * @returns The mappings that should be applied to the target index. + */ +export function deleteTypeMappingsFields( + targetMappings: IndexMapping, + opensearchDashboardsRawConfig?: Config +) { + if (opensearchDashboardsRawConfig?.get('migrations.delete.enabled')) { + const deleteTypes = new Set(opensearchDashboardsRawConfig.get('migrations.delete.types')); + const newProperties = Object.keys(targetMappings.properties) + .filter((key) => !deleteTypes.has(key)) + .reduce((obj, key) => { + return { ...obj, [key]: targetMappings.properties[key] }; + }, {}); + + return { + ...targetMappings, + properties: newProperties, + }; + } + + return targetMappings; +} + /** * Gets the next index name in a sequence, based on specified current index's info. * We're using a numeric counter to create new indices. So, `.opensearch_dashboards_1`, `.opensearch_dashboards_2`, etc @@ -206,6 +253,6 @@ export function disableUnknownTypeMappingFields( */ function nextIndexName(indexName: string, alias: string) { const indexSuffix = (indexName.match(/[\d]+$/) || [])[0]; - const indexNum = parseInt(indexSuffix, 10) || 0; + const indexNum = parseInt(indexSuffix!, 10) || 0; return `${alias}_${indexNum + 1}`; } diff --git a/src/core/server/saved_objects/migrations/core/migration_opensearch_client.test.ts b/src/core/server/saved_objects/migrations/core/migration_opensearch_client.test.ts index 91f11cbd4878..8675f86c10ea 100644 --- a/src/core/server/saved_objects/migrations/core/migration_opensearch_client.test.ts +++ b/src/core/server/saved_objects/migrations/core/migration_opensearch_client.test.ts @@ -76,4 +76,8 @@ describe('MigrationOpenSearchClient', () => { expect(SavedObjectsErrorHelpers.isSavedObjectsClientError(e)).toBe(false); } }); + + it('should have the deleteByQuery method', () => { + expect(client.deleteByQuery).toBeDefined(); + }); }); diff --git a/src/core/server/saved_objects/migrations/core/migration_opensearch_client.ts b/src/core/server/saved_objects/migrations/core/migration_opensearch_client.ts index 7ab77d5a62dd..4cb4fef39de3 100644 --- a/src/core/server/saved_objects/migrations/core/migration_opensearch_client.ts +++ b/src/core/server/saved_objects/migrations/core/migration_opensearch_client.ts @@ -51,6 +51,7 @@ const methods = [ 'search', 'scroll', 'tasks.get', + 'deleteByQuery', ] as const; type MethodName = typeof methods[number]; @@ -77,6 +78,7 @@ export interface MigrationOpenSearchClient { tasks: { get: OpenSearchClient['tasks']['get']; }; + deleteByQuery: OpenSearchClient['deleteByQuery']; } export function createMigrationOpenSearchClient( diff --git a/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.test.ts b/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.test.ts index e65effdd8eaa..0a52b1947f2f 100644 --- a/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.test.ts +++ b/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.test.ts @@ -89,6 +89,12 @@ describe('OpenSearchDashboardsMigrator', () => { const mappings = new OpenSearchDashboardsMigrator(options).getActiveMappings(); expect(mappings).toHaveProperty('properties.workspaces'); }); + + it('text field does not exist in the mappings when the feature is enabled', () => { + const options = mockOptions(false, false, { enabled: true, types: ['text'] }); + const mappings = new OpenSearchDashboardsMigrator(options).getActiveMappings(); + expect(mappings).not.toHaveProperty('properties.text'); + }); }); describe('runMigrations', () => { @@ -159,10 +165,14 @@ type MockedOptions = OpenSearchDashboardsMigratorOptions & { client: ReturnType; }; -const mockOptions = (isWorkspaceEnabled?: boolean, isPermissionControlEnabled?: boolean) => { +const mockOptions = ( + isWorkspaceEnabled?: boolean, + isPermissionControlEnabled?: boolean, + deleteConfig?: { enabled: boolean; types: string[] } +) => { const rawConfig = configMock.create(); rawConfig.get.mockReturnValue(false); - if (isWorkspaceEnabled || isPermissionControlEnabled) { + if (isWorkspaceEnabled || isPermissionControlEnabled || deleteConfig?.enabled) { rawConfig.get.mockReturnValue(true); } rawConfig.get.mockImplementation((path) => { @@ -178,6 +188,18 @@ const mockOptions = (isWorkspaceEnabled?: boolean, isPermissionControlEnabled?: } else { return false; } + } else if (path === 'migrations.delete.enabled') { + if (deleteConfig?.enabled) { + return true; + } else { + return false; + } + } else if (path === 'migrations.delete.types') { + if (deleteConfig?.enabled) { + return deleteConfig?.types; + } else { + return []; + } } else { return false; } @@ -209,6 +231,18 @@ const mockOptions = (isWorkspaceEnabled?: boolean, isPermissionControlEnabled?: }, migrations: {}, }, + { + name: 'testtype3', + hidden: false, + namespaceType: 'single', + indexPattern: 'other-index', + mappings: { + properties: { + name: { type: 'text' }, + }, + }, + migrations: {}, + }, ]), opensearchDashboardsConfig: { enabled: true, @@ -219,6 +253,10 @@ const mockOptions = (isWorkspaceEnabled?: boolean, isPermissionControlEnabled?: pollInterval: 20000, scrollDuration: '10m', skip: false, + delete: { + enabled: rawConfig.get('migrations.delete.enabled'), + types: rawConfig.get('migrations.delete.types'), + }, }, client: opensearchClientMock.createOpenSearchClient(), opensearchDashboardsRawConfig: rawConfig, diff --git a/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.ts b/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.ts index d6c119569a2e..e0e623f20f94 100644 --- a/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.ts +++ b/src/core/server/saved_objects/migrations/opensearch_dashboards/opensearch_dashboards_migrator.ts @@ -187,6 +187,9 @@ export class OpenSearchDashboardsMigrator { index === opensearchDashboardsIndexName ? 'opensearch_dashboards_index_template*' : undefined, + typesToDelete: this.savedObjectsConfig.delete.enabled + ? this.savedObjectsConfig.delete.types + : undefined, convertToAliasScript: indexMap[index].script, opensearchDashboardsRawConfig: this.opensearchDashboardsRawConfig, }); diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts index e6ffaefb8a59..ccf95b21cd45 100644 --- a/src/core/server/saved_objects/saved_objects_config.ts +++ b/src/core/server/saved_objects/saved_objects_config.ts @@ -39,6 +39,19 @@ export const savedObjectsMigrationConfig = { scrollDuration: schema.string({ defaultValue: '15m' }), pollInterval: schema.number({ defaultValue: 1500 }), skip: schema.boolean({ defaultValue: false }), + delete: schema.object( + { + enabled: schema.boolean({ defaultValue: false }), + types: schema.arrayOf(schema.string(), { defaultValue: [] }), + }, + { + validate(value) { + if (value.enabled === true && value.types.length === 0) { + return 'delete types cannot be empty when delete is enabled'; + } + }, + } + ), }), }; diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker index 1755ada630f2..3f17d5e7cc60 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker @@ -77,6 +77,8 @@ opensearch_dashboards_vars=( map.tilemap.options.minZoom map.tilemap.options.subdomains map.tilemap.url + migrations.delete.enabled + migrations.delete.types monitoring.cluster_alerts.email_notifications.email_address monitoring.enabled monitoring.opensearchDashboards.collection.enabled