[esArchiver] restrict from modifying saved objects indexes (elastic#1…

…69852)

## Summary

Related to elastic#161882

The goal is to prevent FTR tests from new esArchives that overrides SO
indexes.

This PR adds the existing archives that re-create Saved Objects indexes
into temporary exception list, located in
`packages/kbn-es-archiver/src/fixtures/override_saved_objects_index/exception_list.json`.
Whenever tests/archives are updated to not modify SO indexes, archive is
expected to be removed from the list (progress can be tracked in
elastic#169075, elastic#168973, elastic#168969, elastic#168926 )

Load action has a check if index is SO index and if the archive is in
the exception list. This will throw error for the new archives, but
still work as usual for the existing ones while teams updating the
tests.

Whenever test is loading archive listed in the exception list, the
following warning message is logged:
```
warn x-pack/test/functional/es_archives/data/search_sessions overrides Saved Objects index(es) and placed temporary in the exception list.
Please fix the archive and remove it from /Users/dmle/github/kibana/packages/kbn-es-archiver/src/fixtures/override_saved_objects_index/exception_list.json.
For more details see: elastic#161882
```

If the test loads a newly added archive that modifies a Saved Object
index (e.g. `.kibana`), esArchiver will throw the error:
```
 Error: esArchiver doesn't support modifying the existing Saved Objects index: '.kibana_1',
 please update its definition in mappings.json
```

packages/kbn-es-archiver/src/actions/load.ts

@@ -30,6 +30,8 @@ import {
   createDefaultSpace,
 } from '../lib';
 
+import soOverrideAllowedList from '../fixtures/override_saved_objects_index/exception_list.json';
+
 // pipe a series of streams into each other so that data and errors
 // flow from the first stream to the last. Errors from the last stream
 // are not listened for
@@ -38,6 +40,16 @@ const pipeline = (...streams: Readable[]) =>
     source.once('error', (error) => dest.destroy(error)).pipe(dest as any)
   );
 
+const warningToUpdateArchive = (path: string) => {
+  return `This test is using '${path}' archive that contains saved object index definitions (in the 'mappings.json').
+This has proven to be a source of conflicts and flakiness, so the goal is to remove support for this feature ASAP. We kindly ask you to
+update your test archives and remove SO index definitions, so that tests use the official saved object indices created by Kibana at startup.
+You can achieve that by simply removing your saved object index definitions from 'mappings.json' (likely removing the file altogether).
+We also recommend migrating existing tests to 'kbnArchiver' whenever possible. After the fix please remove archive path from the exception list:
+${resolve(__dirname, '../fixtures/override_saved_objects_index/exception_list.json')}.
+Find more information here: https://github.com/elastic/kibana/issues/161882`;
+};
+
 export async function loadAction({
   inputDir,
   skipExisting,
@@ -56,6 +68,10 @@ export async function loadAction({
   kbnClient: KbnClient;
 }) {
   const name = relative(REPO_ROOT, inputDir);
+  const isArchiveInExceptionList = soOverrideAllowedList.includes(name);
+  if (isArchiveInExceptionList) {
+    log.warning(warningToUpdateArchive(name));
+  }
   const stats = createStats(name, log);
   const files = prioritizeMappings(await readDirectory(inputDir));
   const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
@@ -80,7 +96,14 @@ export async function loadAction({
 
   await createPromiseFromStreams([
     recordStream,
-    createCreateIndexStream({ client, stats, skipExisting, docsOnly, log }),
+    createCreateIndexStream({
+      client,
+      stats,
+      skipExisting,
+      docsOnly,
+      isArchiveInExceptionList,
+      log,
+    }),
     createIndexDocRecordsStream(client, stats, progress, useCreate),
   ]);
 

packages/kbn-es-archiver/src/fixtures/override_saved_objects_index/exception_list.json

@@ -0,0 +1,35 @@
+[
+  "x-pack/test/functional/es_archives/action_task_params",
+  "x-pack/test/functional/es_archives/actions",
+  "x-pack/test/functional/es_archives/alerting/8_2_0",
+  "x-pack/test/functional/es_archives/alerts",
+  "x-pack/test/functional/es_archives/alerts_legacy/rules",
+  "x-pack/test/functional/es_archives/alerts_legacy/tasks",
+  "x-pack/test/functional/es_archives/cases/default",
+  "x-pack/test/functional/es_archives/cases/migrations/7.11.1",
+  "x-pack/test/functional/es_archives/cases/migrations/7.13.2",
+  "x-pack/test/functional/es_archives/cases/migrations/7.13_user_actions",
+  "x-pack/test/functional/es_archives/cases/migrations/7.16.0_space",
+  "x-pack/test/functional/es_archives/cases/migrations/8.8.0",
+  "x-pack/test/functional/es_archives/data/search_sessions",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/agent_only",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_different_states",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_installed",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/cloned_endpoint_uninstalled",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_disabled",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_malware_enabled",
+  "x-pack/test/functional/es_archives/endpoint/telemetry/endpoint_uninstalled",
+  "x-pack/test/functional/es_archives/event_log_legacy_ids",
+  "x-pack/test/functional/es_archives/event_log_multiple_indicies",
+  "x-pack/test/functional/es_archives/fleet/agents",
+  "x-pack/test/functional/es_archives/lists",
+  "x-pack/test/functional/es_archives/rules_scheduled_task_id/rules",
+  "x-pack/test/functional/es_archives/rules_scheduled_task_id/tasks",
+  "x-pack/test/functional/es_archives/security_solution/import_rule_connector",
+  "x-pack/test/functional/es_archives/security_solution/migrations",
+  "x-pack/test/functional/es_archives/security_solution/resolve_read_rules/7_14",
+  "x-pack/test/functional/es_archives/security_solution/timelines/7.15.0",
+  "x-pack/test/functional/es_archives/security_solution/timelines/7.15.0_space",
+  "x-pack/test/functional/es_archives/task_manager_removed_types",
+  "x-pack/test/functional/es_archives/task_manager_tasks"
+]

packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.mock.ts

@@ -6,7 +6,11 @@
  * Side Public License, v 1.
  */
 
-import type { cleanSavedObjectIndices, deleteSavedObjectIndices } from './kibana_index';
+import type {
+  cleanSavedObjectIndices,
+  deleteSavedObjectIndices,
+  isSavedObjectIndex,
+} from './kibana_index';
 
 export const mockCleanSavedObjectIndices = jest.fn() as jest.MockedFunction<
   typeof cleanSavedObjectIndices
@@ -16,7 +20,12 @@ export const mockDeleteSavedObjectIndices = jest.fn() as jest.MockedFunction<
   typeof deleteSavedObjectIndices
 >;
 
+export const mockIsSavedObjectIndex = jest.fn() as unknown as jest.MockedFunction<
+  typeof isSavedObjectIndex
+>;
+
 jest.mock('./kibana_index', () => ({
   cleanSavedObjectIndices: mockCleanSavedObjectIndices,
   deleteSavedObjectIndices: mockDeleteSavedObjectIndices,
+  isSavedObjectIndex: mockIsSavedObjectIndex,
 }));

packages/kbn-es-archiver/src/lib/indices/create_index_stream.test.ts

@@ -7,6 +7,7 @@
  */
 
 import {
+  mockIsSavedObjectIndex,
   mockCleanSavedObjectIndices,
   mockDeleteSavedObjectIndices,
 } from './create_index_stream.test.mock';
@@ -31,6 +32,7 @@ const chance = new Chance();
 const log = createStubLogger();
 
 beforeEach(() => {
+  mockIsSavedObjectIndex.mockClear();
   mockCleanSavedObjectIndices.mockClear();
   mockDeleteSavedObjectIndices.mockClear();
 });

packages/kbn-es-archiver/src/lib/indices/create_index_stream.ts

@@ -19,7 +19,11 @@ import {
   TASK_MANAGER_SAVED_OBJECT_INDEX,
 } from '@kbn/core-saved-objects-server';
 import { Stats } from '../stats';
-import { cleanSavedObjectIndices, deleteSavedObjectIndices } from './kibana_index';
+import {
+  cleanSavedObjectIndices,
+  deleteSavedObjectIndices,
+  isSavedObjectIndex,
+} from './kibana_index';
 import { deleteIndex } from './delete_index';
 import { deleteDataStream } from './delete_data_stream';
 import { ES_CLIENT_HEADERS } from '../../client_headers';
@@ -37,12 +41,14 @@ export function createCreateIndexStream({
   stats,
   skipExisting = false,
   docsOnly = false,
+  isArchiveInExceptionList = false,
   log,
 }: {
   client: Client;
   stats: Stats;
   skipExisting?: boolean;
   docsOnly?: boolean;
+  isArchiveInExceptionList?: boolean;
   log: ToolingLog;
 }) {
   const skipDocsFromIndices = new Set();
@@ -129,6 +135,16 @@ export function createCreateIndexStream({
       return;
     }
 
+    if (isSavedObjectIndex(index) && !isArchiveInExceptionList) {
+      throw new Error(
+        `'esArchiver' no longer supports defining saved object indices, your archive is modifying '${index}'.
+      The recommendation is to use 'kbnArchiver' to import saved objects in your tests.
+      If you absolutely need to load some non-importable SOs, please stick to the official saved object indices created by Kibana at startup.
+      You can achieve that by simply removing your saved object index definitions from 'mappings.json' (likely removing the file altogether).
+      Find more information here: https://github.com/elastic/kibana/issues/161882`
+      );
+    }
+
     async function attemptToCreate(attemptNumber = 1) {
       try {
         if (isKibana && !kibanaIndicesAlreadyDeleted) {

packages/kbn-es-archiver/src/lib/indices/kibana_index.ts

@@ -85,7 +85,7 @@ export async function migrateSavedObjectIndices(kbnClient: KbnClient) {
 const LEGACY_INDICES_REGEXP = new RegExp(`^(${ALL_SAVED_OBJECT_INDICES.join('|')})(:?_\\d*)?$`);
 const INDICES_REGEXP = new RegExp(`^(${ALL_SAVED_OBJECT_INDICES.join('|')})_(pre)?\\d+.\\d+.\\d+`);
 
-function isSavedObjectIndex(index?: string): index is string {
+export function isSavedObjectIndex(index?: string): index is string {
   return Boolean(index && (LEGACY_INDICES_REGEXP.test(index) || INDICES_REGEXP.test(index)));
 }
 

packages/kbn-es-archiver/tsconfig.json

@@ -8,7 +8,8 @@
     ]
   },
   "include": [
-    "**/*.ts"
+    "**/*.ts",
+    "src/**/*.json"
   ],
   "kbn_references": [
     "@kbn/core-saved-objects-server",

test/functional/fixtures/kbn_archiver/canvas/workpad_pdf_test.json

@@ -0,0 +1,78 @@
+{
+  "attributes":{
+     "@created":"2020-10-21T23:50:26.466Z",
+     "@timestamp":"2020-10-22T23:44:48.387Z",
+     "assets":{},
+     "colors":[
+        "#37988d",
+        "#c19628",
+        "#b83c6f",
+        "#3f9939",
+        "#1785b0",
+        "#ca5f35",
+        "#45bdb0",
+        "#f2bc33",
+        "#e74b8b",
+        "#4fbf48",
+        "#1ea6dc",
+        "#fd7643",
+        "#72cec3",
+        "#f5cc5d",
+        "#ec77a8",
+        "#7acf74",
+        "#4cbce4",
+        "#fd986f",
+        "#a1ded7",
+        "#f8dd91",
+        "#f2a4c5",
+        "#a6dfa2",
+        "#86d2ed",
+        "#fdba9f",
+        "#000000",
+        "#444444",
+        "#777777",
+        "#BBBBBB",
+        "#FFFFFF",
+        "rgba(255,255,255,0)"
+     ],
+     "css":".canvasPage {\n\n}",
+     "height":8,
+     "isWriteable":true,
+     "name":"The Very Cool Workpad for PDF Tests",
+     "page":0,
+     "pages":[
+        {
+           "elements":[
+              {
+                 "expression":"shape \"square\" fill=\"#4cbce4\" border=\"rgba(255,255,255,0)\" borderWidth=0 maintainAspect=false\n| render",
+                 "id":"element-e05237c6-0fda-49e7-84fc-b8573739b515",
+                 "position":{
+                    "angle":0,
+                    "height":18,
+                    "left":-1,
+                    "parent":null,
+                    "top":-1,
+                    "width":15.5
+                 }
+              }
+           ],
+           "groups":[],
+           "id":"page-c0e601aa-1c72-4d4a-b73f-8fb72a9d8d3a",
+           "style":{
+              "background":"#FFF"
+           },
+           "transition":{}
+        }
+     ],
+     "variables":[],
+     "width":8
+  },
+  "coreMigrationVersion":"8.8.0",
+  "id":"workpad-c13808dc-e690-4bab-be06-2073ba071754",
+  "managed":false,
+  "references":[],
+  "type":"canvas-workpad",
+  "typeMigrationVersion":"8.9.0",
+  "updated_at":"2020-10-22T23:44:48.392Z",
+  "version":"WzIsMl0="
+}