Migrations v2 ignore fleet agent events

src/core/server/saved_objects/migrations/core/elastic_index.test.ts

@@ -414,11 +414,18 @@ describe('ElasticIndex', () => {
           size: 100,
           query: {
             bool: {
-              must_not: {
-                term: {
-                  type: 'fleet-agent-events',
+              must_not: [
+                {
+                  term: {
+                    type: 'fleet-agent-events',
+                  },
                 },
-              },
+                {
+                  term: {
+                    type: 'tsvb-validation-telemetry',
+                  },
+                },
+              ],
             },
           },
         },

src/core/server/saved_objects/migrations/core/elastic_index.ts

@@ -70,16 +70,19 @@ export function reader(
   let scrollId: string | undefined;
 
   // When migrating from the outdated index we use a read query which excludes
-  // saved objects which are no longer used. These saved objects will still be
-  // kept in the outdated index for backup purposes, but won't be availble in
-  // the upgraded index.
-  const excludeUnusedTypes = {
+  // saved object types which are no longer used. These saved objects will
+  // still be kept in the outdated index for backup purposes, but won't be
+  // availble in the upgraded index.
+  const EXCLUDE_UNUSED_TYPES = [
+    'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
+    'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617
+  ];
+
+  const excludeUnusedTypesQuery = {
     bool: {
-      must_not: {
-        term: {
-          type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
-        },
-      },
+      must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({
+        term: { type },
+      })),
     },
   };
 
@@ -92,7 +95,7 @@ export function reader(
       : client.search<SearchResponse<SavedObjectsRawDocSource>>({
           body: {
             size: batchSize,
-            query: excludeUnusedTypes,
+            query: excludeUnusedTypesQuery,
           },
           index,
           scroll,

src/core/server/saved_objects/migrations/kibana/kibana_migrator.test.ts

@@ -321,7 +321,7 @@ describe('KibanaMigrator', () => {
         options.client.tasks.get.mockReturnValue(
           elasticsearchClientMock.createSuccessTransportRequestPromise({
             completed: true,
-            error: { type: 'elatsicsearch_exception', reason: 'task failed with an error' },
+            error: { type: 'elasticsearch_exception', reason: 'task failed with an error' },
             failures: [],
             task: { description: 'task description' } as any,
           })
@@ -331,11 +331,11 @@ describe('KibanaMigrator', () => {
         migrator.prepareMigrations();
         await expect(migrator.runMigrations()).rejects.toMatchInlineSnapshot(`
                 [Error: Unable to complete saved object migrations for the [.my-index] index. Error: Reindex failed with the following error:
-                {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}]
+                {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}]
               `);
         expect(loggingSystemMock.collect(options.logger).error[0][0]).toMatchInlineSnapshot(`
           [Error: Reindex failed with the following error:
-          {"_tag":"Some","value":{"type":"elatsicsearch_exception","reason":"task failed with an error"}}]
+          {"_tag":"Some","value":{"type":"elasticsearch_exception","reason":"task failed with an error"}}]
         `);
       });
     });

src/core/server/saved_objects/migrationsv2/actions/index.test.ts

@@ -85,7 +85,8 @@ describe('actions', () => {
         'my_source_index',
         'my_target_index',
         Option.none,
-        false
+        false,
+        Option.none
       );
       try {
         await task();

src/core/server/saved_objects/migrationsv2/actions/index.ts

@@ -14,6 +14,7 @@ import { errors as EsErrors } from '@elastic/elasticsearch';
 import type { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { flow } from 'fp-ts/lib/function';
+import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl';
 import { ElasticsearchClient } from '../../../elasticsearch';
 import { IndexMapping } from '../../mappings';
 import { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '../../serialization';
@@ -436,7 +437,12 @@ export const reindex = (
   sourceIndex: string,
   targetIndex: string,
   reindexScript: Option.Option<string>,
-  requireAlias: boolean
+  requireAlias: boolean,
+  /* When reindexing we use a source query to exclude saved objects types which
+   * are no longer used. These saved objects will still be kept in the outdated
+   * index for backup purposes, but won't be availble in the upgraded index.
+   */
+  unusedTypesToExclude: Option.Option<string[]>
 ): TaskEither.TaskEither<RetryableEsClientError, ReindexResponse> => () => {
   return client
     .reindex({
@@ -450,6 +456,15 @@ export const reindex = (
           index: sourceIndex,
           // Set reindex batch size
           size: BATCH_SIZE,
+          // Exclude saved object types
+          query: Option.fold<string[], QueryContainer | undefined>(
+            () => undefined,
+            (types) => ({
+              bool: {
+                must_not: types.map((type) => ({ term: { type } })),
+              },
+            })
+          )(unusedTypesToExclude),
         },
         dest: {
           index: targetIndex,

src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts

@@ -66,7 +66,8 @@ describe('migration actions', () => {
       { _source: { title: 'doc 1' } },
       { _source: { title: 'doc 2' } },
       { _source: { title: 'doc 3' } },
-      { _source: { title: 'saved object 4' } },
+      { _source: { title: 'saved object 4', type: 'another_unused_type' } },
+      { _source: { title: 'f-agent-event 5', type: 'f_agent_event' } },
     ] as unknown) as SavedObjectsRawDoc[];
     await bulkOverwriteTransformedDocuments(client, 'existing_index_with_docs', sourceDocs)();
 
@@ -343,7 +344,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -364,6 +366,37 @@ describe('migration actions', () => {
           "doc 2",
           "doc 3",
           "saved object 4",
+          "f-agent-event 5",
+        ]
+      `);
+    });
+    it('resolves right and excludes all unusedTypesToExclude documents', async () => {
+      const res = (await reindex(
+        client,
+        'existing_index_with_docs',
+        'reindex_target_excluded_docs',
+        Option.none,
+        false,
+        Option.some(['f_agent_event', 'another_unused_type'])
+      )()) as Either.Right<ReindexResponse>;
+      const task = waitForReindexTask(client, res.right.taskId, '10s');
+      await expect(task()).resolves.toMatchInlineSnapshot(`
+        Object {
+          "_tag": "Right",
+          "right": "reindex_succeeded",
+        }
+      `);
+
+      const results = ((await searchForOutdatedDocuments(client, {
+        batchSize: 1000,
+        targetIndex: 'reindex_target_excluded_docs',
+        outdatedDocumentsQuery: undefined,
+      })()) as Either.Right<SearchResponse>).right.outdatedDocuments;
+      expect(results.map((doc) => doc._source.title)).toMatchInlineSnapshot(`
+        Array [
+          "doc 1",
+          "doc 2",
+          "doc 3",
         ]
       `);
     });
@@ -374,7 +407,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_2',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -394,6 +428,7 @@ describe('migration actions', () => {
           "doc 2_updated",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -405,7 +440,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_3',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       let task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -421,7 +457,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_3',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -443,6 +480,7 @@ describe('migration actions', () => {
           "doc 2_updated",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -469,7 +507,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_4',
         Option.some(`ctx._source.title = ctx._source.title + '_updated'`),
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -491,6 +530,7 @@ describe('migration actions', () => {
           "doc 2",
           "doc 3_updated",
           "saved object 4_updated",
+          "f-agent-event 5_updated",
         ]
       `);
     });
@@ -517,7 +557,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_5',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, reindexTaskId, '10s');
 
@@ -551,7 +592,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_6',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, reindexTaskId, '10s');
 
@@ -571,7 +613,8 @@ describe('migration actions', () => {
         'no_such_index',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`
@@ -591,7 +634,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'existing_index_with_write_block',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -612,7 +656,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'existing_index_with_write_block',
         Option.none,
-        true
+        true,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '10s');
@@ -633,7 +678,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
 
       const task = waitForReindexTask(client, res.right.taskId, '0s');
@@ -659,7 +705,8 @@ describe('migration actions', () => {
         'existing_index_with_docs',
         'reindex_target_7',
         Option.none,
-        false
+        false,
+        Option.none
       )()) as Either.Right<ReindexResponse>;
       await waitForReindexTask(client, res.right.taskId, '10s')();
 
@@ -714,7 +761,7 @@ describe('migration actions', () => {
         targetIndex: 'existing_index_with_docs',
         outdatedDocumentsQuery: undefined,
       })()) as Either.Right<SearchResponse>).right.outdatedDocuments;
-      expect(resultsWithoutQuery.length).toBe(4);
+      expect(resultsWithoutQuery.length).toBe(5);
     });
     it('resolves with _id, _source, _seq_no and _primary_term', async () => {
       expect.assertions(1);

src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts

@@ -249,6 +249,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -310,6 +317,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -456,6 +470,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },
@@ -512,6 +533,13 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
+            "unusedTypesToExclude": Object {
+              "_tag": "Some",
+              "value": Array [
+                "fleet-agent-events",
+                "tsvb-validation-telemetry",
+              ],
+            },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
           },