Attempt to fix flaky tests after split migration (#159397)

## Summary Addresses root cause of #158918 Underlying cause is that _esArchiver_ is messing up with the SO indices whilst Kibana is already running. This can cause some asynchronous calls made by Kibana (e.g. `GET /.kibana_8.8.0/telemetry:telemetry`) to hit ES at the exact time where the underlying SO indices are **just** recreated, causing the error described in the related issue. The idea of the fix is to delete `mappings.json`, used by _esArchiver_ to create the SO indices. This way, _esArchiver_ will use existing SO indices instead (aka the "official" ones, created by Kibana at startup), thus avoiding the problem altogether. As a side effect: - Documents in `data.json` must be updated so that they are correctly inserted. - The different FTR tests must make sure the SO indices are empty before inserting those documents (done in the `before(), beforeEach()` statements).
elastic · Jul 7, 2023 · 79f7bb4 · 79f7bb4
1 parent d9c0c55
commit 79f7bb4
Show file tree

Hide file tree

Showing 9 changed files with 118 additions and 321 deletions.
diff --git a/packages/kbn-es-archiver/src/actions/unload.ts b/packages/kbn-es-archiver/src/actions/unload.ts
@@ -46,7 +46,7 @@ export async function unloadAction({
     await createPromiseFromStreams([
       createReadStream(resolve(inputDir, filename)) as Readable,
       ...createParseArchiveStreams({ gzip: isGzip(filename) }),
-      createFilterRecordsStream((record) => ['index', 'data_stream'].includes(record.type)),
+      createFilterRecordsStream((record) => ['index', 'data_stream', 'doc'].includes(record.type)),
       createDeleteIndexStream(client, stats, log),
     ] as [Readable, ...Writable[]]);
   }

diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.test.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.test.ts
@@ -20,6 +20,7 @@ import {
   createStubIndexRecord,
   createStubDataStreamRecord,
   createStubLogger,
+  createStubDocRecord,
 } from './__mocks__/stubs';
 
 const log = createStubLogger();
@@ -79,4 +80,49 @@ describe('esArchiver: createDeleteIndexStream()', () => {
       name: 'foo-template',
     });
   });
+
+  describe('saved object cleanup', () => {
+    describe('when saved object documents are found', () => {
+      it('cleans the corresponding saved object indices', async () => {
+        const client = createStubClient();
+        const stats = createStubStats();
+        await createPromiseFromStreams([
+          createListStream([
+            createStubDocRecord('.kibana_task_manager', 1),
+            createStubDocRecord('.kibana_alerting_cases', 2),
+            createStubDocRecord('.kibana', 3),
+          ]),
+          createDeleteIndexStream(client, stats, log),
+        ]);
+
+        expect(mockCleanSavedObjectIndices).toHaveBeenCalledTimes(2);
+
+        expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
+          1,
+          expect.objectContaining({ index: '.kibana_task_manager' })
+        );
+        expect(mockCleanSavedObjectIndices).toHaveBeenNthCalledWith(
+          2,
+          expect.not.objectContaining({ index: expect.any(String) })
+        );
+      });
+    });
+
+    describe('when saved object documents are not found', () => {
+      it('does not clean any indices', async () => {
+        const client = createStubClient();
+        const stats = createStubStats();
+        await createPromiseFromStreams([
+          createListStream([
+            createStubDocRecord('.foo', 1),
+            createStubDocRecord('.bar', 2),
+            createStubDocRecord('.baz', 3),
+          ]),
+          createDeleteIndexStream(client, stats, log),
+        ]);
+
+        expect(mockCleanSavedObjectIndices).not.toHaveBeenCalled();
+      });
+    });
+  });
 });
diff --git a/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts b/packages/kbn-es-archiver/src/lib/indices/delete_index_stream.ts
@@ -60,7 +60,22 @@ export function createDeleteIndexStream(client: Client, stats: Stats, log: Tooli
           await deleteDataStream(client, dataStream, name);
           stats.deletedDataStream(dataStream, name);
         } else {
-          this.push(record);
+          if (record.type === 'doc') {
+            const index = record.value.index;
+            if (index.startsWith(TASK_MANAGER_SAVED_OBJECT_INDEX)) {
+              if (!kibanaTaskManagerIndexAlreadyCleaned) {
+                await cleanSavedObjectIndices({ client, stats, index, log });
+                kibanaTaskManagerIndexAlreadyCleaned = true;
+                log.debug(`Cleaned saved object index [${index}]`);
+              }
+            } else if (index.startsWith(MAIN_SAVED_OBJECT_INDEX)) {
+              if (!kibanaIndicesAlreadyCleaned) {
+                await cleanSavedObjectIndices({ client, stats, log });
+                kibanaIndicesAlreadyCleaned = kibanaTaskManagerIndexAlreadyCleaned = true;
+                log.debug(`Cleaned all saved object indices`);
+              }
+            }
+          }
         }
         callback();
       } catch (err) {

diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts
@@ -121,6 +121,7 @@ export async function cleanSavedObjectIndices({
     const resp = await client.deleteByQuery(
       {
         index,
+        refresh: true,
         body: {
           query: {
             bool: {
@@ -139,7 +140,7 @@ export async function cleanSavedObjectIndices({
       }
     );
 
-    if (resp.total !== resp.deleted) {
+    if (resp.total !== resp.deleted && resp.total && resp.total > 1) {
       log.warning(
         'delete by query deleted %d of %d total documents, trying again',
         resp.deleted,

diff --git a/...ck/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json b/...ck/test/spaces_api_integration/common/fixtures/es_archiver/saved_objects/spaces/data.json
@@ -1,22 +1,3 @@
-{
-  "type": "_doc",
-  "value": {
-    "id": "space:default",
-    "index": ".kibana",
-    "source": {
-      "space": {
-        "_reserved": true,
-        "description": "This is the default space",
-        "disabledFeatures": [],
-        "name": "Default Space"
-      },
-      "type": "space",
-      "updated_at": "2017-09-21T18:49:16.270Z"
-    },
-    "type": "_doc"
-  }
-}
-
 {
   "type": "_doc",
   "value": {
@@ -29,7 +10,11 @@
         "name": "Space 1"
       },
       "type": "space",
-      "updated_at": "2017-09-21T18:49:16.270Z"
+      "updated_at": "2017-09-21T18:49:16.270Z",
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "6.6.0",
+      "managed": false,
+      "references": []
     },
     "type": "_doc"
   }
@@ -47,7 +32,11 @@
         "name": "Space 2"
       },
       "type": "space",
-      "updated_at": "2017-09-21T18:49:16.270Z"
+      "updated_at": "2017-09-21T18:49:16.270Z",
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "6.6.0",
+      "managed": false,
+      "references": []
     },
     "type": "_doc"
   }
@@ -120,7 +109,9 @@
       }],
       "type": "dashboard",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["default"]
+      "namespaces": ["default"],
+      "typeMigrationVersion": "8.7.0",
+      "coreMigrationVersion": "8.8.0"
     },
     "type": "_doc"
   }
@@ -149,7 +140,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["default"]
+      "namespaces": ["default"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -178,7 +171,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["default"]
+      "namespaces": ["default"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -208,7 +203,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["default"]
+      "namespaces": ["default"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -244,7 +241,9 @@
       ],
       "type": "dashboard",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["space_1"]
+      "namespaces": ["space_1"],
+      "typeMigrationVersion": "8.7.0",
+      "coreMigrationVersion": "8.8.0"
     },
     "type": "_doc"
   }
@@ -273,7 +272,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["space_1"]
+      "namespaces": ["space_1"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -302,7 +303,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["space_1"]
+      "namespaces": ["space_1"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -332,7 +335,9 @@
       ],
       "type": "visualization",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["space_1"]
+      "namespaces": ["space_1"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.5.0"
     },
     "type": "_doc"
   }
@@ -351,7 +356,9 @@
       "references": [],
       "type": "index-pattern",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["default"]
+      "namespaces": ["default"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.0.0"
     },
     "type": "_doc"
   }
@@ -370,7 +377,9 @@
       "references": [],
       "type": "index-pattern",
       "updated_at": "2017-09-21T18:49:16.270Z",
-      "namespaces": ["space_1"]
+      "namespaces": ["space_1"],
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.0.0"
     },
     "type": "_doc"
   }
@@ -409,7 +418,9 @@
         "targetNamespace": "default",
         "targetType": "sharedtype",
         "targetId": "default_only"
-      }
+      },
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.2.0"
     }
   }
 }
@@ -505,7 +516,9 @@
         "targetNamespace": "space_2",
         "targetType": "sharedtype",
         "targetId": "space_2_only"
-      }
+      },
+      "coreMigrationVersion": "8.8.0",
+      "typeMigrationVersion": "8.2.0"
     }
   }
 }