Merge branch 'main' into serverSideBatching

.lycheeignore

@@ -54,6 +54,7 @@ my(other)?domain\.com
 
 
 # External urls
+adobe.com
 https://connectionurl.com/
 https://www.hostedgraphite.com/
 http://google.com/

CHANGELOG.md

@@ -109,6 +109,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Multiple Datasource] Support multi data source in Region map ([#6654](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6654))
 - Add `rightNavigationButton` component in chrome service for applications to register and add dev tool to top right navigation. ([#6553](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6553))
 - Enable UI Metric Collector to collect UI Metrics and Application Usage ([#6203](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6203))
+- Add `opensearchDashboards.futureNavigation` config to control dev tool top right nav button. ([#6712](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6712))
 
 ### 🐛 Bug Fixes
 

changelogs/fragments/6443.yml

@@ -0,0 +1,2 @@
+feat:
+- Adds `migrations.delete` to delete saved objects by type during a migration ([#6443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6443))

changelogs/fragments/6571.yml

@@ -0,0 +1,2 @@
+refactor:
+- discover data selector enhancement and refactoring ([#6571](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6571))

changelogs/fragments/6683.yml

@@ -0,0 +1,2 @@
+feat:
+- Optimize scrolling behavior of Discover table ([#6683](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6683))

changelogs/fragments/6712.yml

@@ -0,0 +1,2 @@
+feat:
+- Add `opensearchDashboards.futureNavigation` config to control dev tool top right nav button. ([#6712](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6712))

changelogs/fragments/6722.yml

@@ -0,0 +1,2 @@
+fix:
+- Test failures related to #6443 ([#6722](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6722))

config/opensearch_dashboards.yml

@@ -303,6 +303,9 @@
 # Set the value of this setting to false to hide the help menu link to the OpenSearch Dashboards user survey
 # opensearchDashboards.survey.url: "https://survey.opensearch.org"
 
+# @experimental Set the value of this setting to display navigation updates, including dev tool top right navigation
+# opensearchDashboards.futureNavigation: false
+
 # Set the value of this setting to true to enable plugin augmentation on Dashboard
 # vis_augmenter.pluginAugmentationEnabled: true
 
@@ -314,6 +317,12 @@
 # Set the value to true to enable workspace feature
 # workspace.enabled: false
 
+# Optional settings to specify saved object types to be deleted during migration.
+# This feature can help address compatibility issues that may arise during the migration of saved objects, such as types defined by legacy applications.
+# Please note, using this feature carries a risk. Deleting saved objects during migration could potentially lead to unintended data loss. Use with caution.
+# migrations.delete.enabled: false
+# migrations.delete.types: []
+
 # Set the value to true to enable Ui Metric Collectors in Usage Collector
 # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
-# usageCollection.uiMetric.enabled: false
+# usageCollection.uiMetric.enabled: false

src/core/public/chrome/ui/header/right_navigation_button.tsx

@@ -6,8 +6,12 @@
 import { EuiHeaderSectionItemButton, EuiIcon } from '@elastic/eui';
 import React, { useMemo } from 'react';
 import { CoreStart } from '../../..';
-
 import { isModifiedOrPrevented } from './nav_link';
+
+/**
+ * This component is used for application to render top right navigation button in header.
+ */
+
 export interface RightNavigationButtonProps {
   application: CoreStart['application'];
   http: CoreStart['http'];
@@ -16,6 +20,9 @@ export interface RightNavigationButtonProps {
   title: string;
 }
 
+/**
+ * @experimental this class is experimental and might change in future releases.
+ */
 export const RightNavigationButton = ({
   application,
   http,

src/core/server/mocks.ts

@@ -80,6 +80,7 @@ export function pluginInitializerContextConfigMock<T>(config: T) {
       configIndex: '.opensearch_dashboards_config_tests',
       autocompleteTerminateAfter: duration(100000),
       autocompleteTimeout: duration(1000),
+      futureNavigation: false,
     },
     opensearch: {
       shardTimeout: duration('30s'),

src/core/server/opensearch_dashboards_config.ts

@@ -91,6 +91,7 @@ export const config = {
         defaultValue: 'https://survey.opensearch.org',
       }),
     }),
+    futureNavigation: schema.boolean({ defaultValue: false }),
   }),
   deprecations,
 };

src/core/server/plugins/plugin_context.test.ts

@@ -101,6 +101,7 @@ describe('createPluginInitializerContext', () => {
         configIndex: '.opensearch_dashboards_config',
         autocompleteTerminateAfter: duration(100000),
         autocompleteTimeout: duration(1000),
+        futureNavigation: false,
       },
       opensearch: {
         shardTimeout: duration(30, 's'),

src/core/server/plugins/types.ts

@@ -292,6 +292,7 @@ export const SharedGlobalConfigKeys = {
     'configIndex',
     'autocompleteTerminateAfter',
     'autocompleteTimeout',
+    'futureNavigation',
   ] as const,
   opensearch: ['shardTimeout', 'requestTimeout', 'pingTimeout'] as const,
   path: ['data'] as const,

src/core/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -435,6 +435,228 @@ describe('IndexMigrator', () => {
     });
   });
 
+  test('deletes saved objects by type if configured', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              delete_type: { properties: { type: deleteType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
+  test('retains saved objects by type if delete is not enabled', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return false;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              delete_type: { properties: { type: deleteType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            delete_type: { dynamic: false, properties: {} },
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
+  test('retains saved objects by type if delete types does not exist', async () => {
+    const { client } = testOpts;
+
+    const deleteType = 'delete_type';
+    const retainType = 'retain_type';
+
+    const rawConfig = configMock.create();
+    rawConfig.get.mockImplementation((path) => {
+      if (path === 'migrations.delete.enabled') {
+        return true;
+      }
+      if (path === 'migrations.delete.types') {
+        return [deleteType];
+      }
+    });
+    testOpts.opensearchDashboardsRawConfig = rawConfig;
+
+    testOpts.mappingProperties = { foo: { type: 'text' } as any };
+
+    withIndex(client, {
+      index: {
+        '.kibana_1': {
+          aliases: {},
+          mappings: {
+            properties: {
+              retain_type: { properties: { type: retainType } },
+            },
+          },
+        },
+      },
+    });
+
+    await new IndexMigrator(testOpts).migrate();
+
+    expect(client.indices.create).toHaveBeenCalledWith({
+      body: {
+        mappings: {
+          dynamic: 'strict',
+          _meta: {
+            migrationMappingPropertyHashes: {
+              foo: '625b32086eb1d1203564cf85062dd22e',
+              migrationVersion: '4a1746014a75ade3a714e1db5763276f',
+              namespace: '2f4316de49999235636386fe51dc06c1',
+              namespaces: '2f4316de49999235636386fe51dc06c1',
+              originId: '2f4316de49999235636386fe51dc06c1',
+              references: '7997cf5a56cc02bdc9c93361bde732b0',
+              type: '2f4316de49999235636386fe51dc06c1',
+              updated_at: '00da57df13e94e9d98437d13ace4bfe0',
+            },
+          },
+          properties: {
+            retain_type: { dynamic: false, properties: {} },
+            foo: { type: 'text' },
+            migrationVersion: { dynamic: 'true', type: 'object' },
+            namespace: { type: 'keyword' },
+            namespaces: { type: 'keyword' },
+            originId: { type: 'keyword' },
+            type: { type: 'keyword' },
+            updated_at: { type: 'date' },
+            references: {
+              type: 'nested',
+              properties: {
+                name: { type: 'keyword' },
+                type: { type: 'keyword' },
+                id: { type: 'keyword' },
+              },
+            },
+          },
+        },
+        settings: { number_of_shards: 1, auto_expand_replicas: '0-1' },
+      },
+      index: '.kibana_2',
+    });
+  });
+
   test('points the alias at the dest index', async () => {
     const { client } = testOpts;
 

src/core/server/saved_objects/migrations/core/index_migrator.ts

@@ -28,6 +28,7 @@
  * under the License.
  */
 
+import { DeleteByQueryRequest } from '@opensearch-project/opensearch/api/types';
 import { diffMappings } from './build_active_mappings';
 import * as Index from './opensearch_index';
 import { migrateRawDocs } from './migrate_raw_docs';
@@ -123,6 +124,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
   const { client, alias, source, dest, log } = context;
 
   await deleteIndexTemplates(context);
+  await deleteSavedObjectsByType(context);
 
   log.info(`Creating index ${dest.indexName}.`);
 
@@ -171,6 +173,33 @@ async function deleteIndexTemplates({ client, log, obsoleteIndexTemplatePattern
   return Promise.all(templateNames.map((name) => client.indices.deleteTemplate({ name: name! })));
 }
 
+/**
+ * Delete saved objects by type. If migrations.delete.types is specified,
+ * any saved objects that matches that type will be deleted.
+ */
+async function deleteSavedObjectsByType(context: Context) {
+  const { client, source, log, typesToDelete } = context;
+  if (!source.exists || !typesToDelete || typesToDelete.length === 0) {
+    return;
+  }
+
+  log.info(`Removing saved objects of types: ${typesToDelete.join(', ')}`);
+  const params = {
+    index: source.indexName,
+    body: {
+      query: {
+        bool: {
+          should: [...typesToDelete.map((type) => ({ term: { type } }))],
+        },
+      },
+    },
+    conflicts: 'proceed',
+    refresh: true,
+  } as DeleteByQueryRequest;
+  log.debug(`Delete by query params: ${JSON.stringify(params)}`);
+  return client.deleteByQuery(params);
+}
+
 /**
  * Moves all docs from sourceIndex to destIndex, migrating each as necessary.
  * This moves documents from the concrete index, rather than the alias, to prevent