Exclude non-persisted sessions from SO migration (elastic#96938)

rudolf · Apr 13, 2021 · bfd5b7b · bfd5b7b
1 parent b9c4d24
commit bfd5b7b
Show file tree

Hide file tree

Showing 10 changed files with 254 additions and 65 deletions.
diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts
@@ -425,6 +425,22 @@ describe('ElasticIndex', () => {
                     type: 'tsvb-validation-telemetry',
                   },
                 },
+                {
+                  bool: {
+                    must: [
+                      {
+                        match: {
+                          type: 'search-session',
+                        },
+                      },
+                      {
+                        match: {
+                          'search-session.persisted': false,
+                        },
+                      },
+                    ],
+                  },
+                },
               ],
             },
           },

diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts
@@ -29,6 +29,46 @@ export interface FullIndexInfo {
   mappings: IndexMapping;
 }
 
+// When migrating from the outdated index we use a read query which excludes
+// saved objects which are no longer used. These saved objects will still be
+// kept in the outdated index for backup purposes, but won't be availble in
+// the upgraded index.
+export const excludeUnusedTypesQuery: estypes.QueryContainer = {
+  bool: {
+    must_not: [
+      // https://github.com/elastic/kibana/issues/91869
+      {
+        term: {
+          type: 'fleet-agent-events',
+        },
+      },
+      // https://github.com/elastic/kibana/issues/95617
+      {
+        term: {
+          type: 'tsvb-validation-telemetry',
+        },
+      },
+      // https://github.com/elastic/kibana/issues/96131
+      {
+        bool: {
+          must: [
+            {
+              match: {
+                type: 'search-session',
+              },
+            },
+            {
+              match: {
+                'search-session.persisted': false,
+              },
+            },
+          ],
+        },
+      },
+    ],
+  },
+};
+
 /**
  * A slight enhancement to indices.get, that adds indexName, and validates that the
  * index mappings are somewhat what we expect.
@@ -69,23 +109,6 @@ export function reader(
   const scroll = scrollDuration;
   let scrollId: string | undefined;
 
-  // When migrating from the outdated index we use a read query which excludes
-  // saved object types which are no longer used. These saved objects will
-  // still be kept in the outdated index for backup purposes, but won't be
-  // availble in the upgraded index.
-  const EXCLUDE_UNUSED_TYPES = [
-    'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
-    'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617
-  ];
-
-  const excludeUnusedTypesQuery = {
-    bool: {
-      must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({
-        term: { type },
-      })),
-    },
-  };
-
   const nextBatch = () =>
     scrollId !== undefined
       ? client.scroll<SearchResponse<SavedObjectsRawDocSource>>({

diff --git a/src/core/server/saved_objects/migrations/core/index.ts b/src/core/server/saved_objects/migrations/core/index.ts
@@ -14,3 +14,4 @@ export type { LogFn, SavedObjectsMigrationLogger } from './migration_logger';
 export type { MigrationResult, MigrationStatus } from './migration_coordinator';
 export { createMigrationEsClient } from './migration_es_client';
 export type { MigrationEsClient } from './migration_es_client';
+export { excludeUnusedTypesQuery } from './elastic_index';
diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts
@@ -14,7 +14,6 @@ import { errors as EsErrors } from '@elastic/elasticsearch';
 import type { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { flow } from 'fp-ts/lib/function';
-import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl';
 import { ElasticsearchClient } from '../../../elasticsearch';
 import { IndexMapping } from '../../mappings';
 import { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '../../serialization';
@@ -440,9 +439,9 @@ export const reindex = (
   requireAlias: boolean,
   /* When reindexing we use a source query to exclude saved objects types which
    * are no longer used. These saved objects will still be kept in the outdated
-   * index for backup purposes, but won't be availble in the upgraded index.
+   * index for backup purposes, but won't be available in the upgraded index.
    */
-  unusedTypesToExclude: Option.Option<string[]>
+  unusedTypesQuery: Option.Option<estypes.QueryContainer>
 ): TaskEither.TaskEither<RetryableEsClientError, ReindexResponse> => () => {
   return client
     .reindex({
@@ -457,14 +456,10 @@ export const reindex = (
           // Set reindex batch size
           size: BATCH_SIZE,
           // Exclude saved object types
-          query: Option.fold<string[], QueryContainer | undefined>(
+          query: Option.fold<estypes.QueryContainer, estypes.QueryContainer | undefined>(
             () => undefined,
-            (types) => ({
-              bool: {
-                must_not: types.map((type) => ({ term: { type } })),
-              },
-            })
-          )(unusedTypesToExclude),
+            (query) => query
+          )(unusedTypesQuery),
         },
         dest: {
           index: targetIndex,

diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts
@@ -416,14 +416,20 @@ describe('migration actions', () => {
         ]
       `);
     });
-    it('resolves right and excludes all unusedTypesToExclude documents', async () => {
+    it('resolves right and excludes all documents not matching the unusedTypesQuery', async () => {
       const res = (await reindex(
         client,
         'existing_index_with_docs',
         'reindex_target_excluded_docs',
         Option.none,
         false,
-        Option.some(['f_agent_event', 'another_unused_type'])
+        Option.of({
+          bool: {
+            must_not: ['f_agent_event', 'another_unused_type'].map((type) => ({
+              term: { type },
+            })),
+          },
+        })
       )()) as Either.Right<ReindexResponse>;
       const task = waitForReindexTask(client, res.right.taskId, '10s');
       await expect(task()).resolves.toMatchInlineSnapshot(`

diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts
@@ -254,12 +254,40 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
-            "unusedTypesToExclude": Object {
+            "unusedTypesQuery": Object {
               "_tag": "Some",
-              "value": Array [
-                "fleet-agent-events",
-                "tsvb-validation-telemetry",
-              ],
+              "value": Object {
+                "bool": Object {
+                  "must_not": Array [
+                    Object {
+                      "term": Object {
+                        "type": "fleet-agent-events",
+                      },
+                    },
+                    Object {
+                      "term": Object {
+                        "type": "tsvb-validation-telemetry",
+                      },
+                    },
+                    Object {
+                      "bool": Object {
+                        "must": Array [
+                          Object {
+                            "match": Object {
+                              "type": "search-session",
+                            },
+                          },
+                          Object {
+                            "match": Object {
+                              "search-session.persisted": false,
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
             },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
@@ -322,12 +350,40 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
-            "unusedTypesToExclude": Object {
+            "unusedTypesQuery": Object {
               "_tag": "Some",
-              "value": Array [
-                "fleet-agent-events",
-                "tsvb-validation-telemetry",
-              ],
+              "value": Object {
+                "bool": Object {
+                  "must_not": Array [
+                    Object {
+                      "term": Object {
+                        "type": "fleet-agent-events",
+                      },
+                    },
+                    Object {
+                      "term": Object {
+                        "type": "tsvb-validation-telemetry",
+                      },
+                    },
+                    Object {
+                      "bool": Object {
+                        "must": Array [
+                          Object {
+                            "match": Object {
+                              "type": "search-session",
+                            },
+                          },
+                          Object {
+                            "match": Object {
+                              "search-session.persisted": false,
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
             },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
@@ -475,12 +531,40 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
-            "unusedTypesToExclude": Object {
+            "unusedTypesQuery": Object {
               "_tag": "Some",
-              "value": Array [
-                "fleet-agent-events",
-                "tsvb-validation-telemetry",
-              ],
+              "value": Object {
+                "bool": Object {
+                  "must_not": Array [
+                    Object {
+                      "term": Object {
+                        "type": "fleet-agent-events",
+                      },
+                    },
+                    Object {
+                      "term": Object {
+                        "type": "tsvb-validation-telemetry",
+                      },
+                    },
+                    Object {
+                      "bool": Object {
+                        "must": Array [
+                          Object {
+                            "match": Object {
+                              "type": "search-session",
+                            },
+                          },
+                          Object {
+                            "match": Object {
+                              "search-session.persisted": false,
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
             },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",
@@ -538,12 +622,40 @@ describe('migrationsStateActionMachine', () => {
                 },
               },
             },
-            "unusedTypesToExclude": Object {
+            "unusedTypesQuery": Object {
               "_tag": "Some",
-              "value": Array [
-                "fleet-agent-events",
-                "tsvb-validation-telemetry",
-              ],
+              "value": Object {
+                "bool": Object {
+                  "must_not": Array [
+                    Object {
+                      "term": Object {
+                        "type": "fleet-agent-events",
+                      },
+                    },
+                    Object {
+                      "term": Object {
+                        "type": "tsvb-validation-telemetry",
+                      },
+                    },
+                    Object {
+                      "bool": Object {
+                        "must": Array [
+                          Object {
+                            "match": Object {
+                              "type": "search-session",
+                            },
+                          },
+                          Object {
+                            "match": Object {
+                              "search-session.persisted": false,
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  ],
+                },
+              },
             },
             "versionAlias": ".my-so-index_7.11.0",
             "versionIndex": ".my-so-index_7.11.0_001",