From 90eae6cfff8ffe1f1239756f626e5d52e06d0a6e Mon Sep 17 00:00:00 2001 From: pgayvallet Date: Tue, 13 Apr 2021 12:15:06 +0200 Subject: [PATCH] Exclude non-persisted sessions from SO migration --- .../migrations/core/elastic_index.test.ts | 16 ++ .../migrations/core/elastic_index.ts | 57 +++++-- .../saved_objects/migrations/core/index.ts | 1 + .../migrationsv2/actions/index.ts | 15 +- .../integration_tests/actions.test.ts | 10 +- .../migrations_state_action_machine.test.ts | 152 +++++++++++++++--- .../saved_objects/migrationsv2/model.test.ts | 50 +++++- .../saved_objects/migrationsv2/model.ts | 9 +- .../server/saved_objects/migrationsv2/next.ts | 4 +- .../saved_objects/migrationsv2/types.ts | 5 +- 10 files changed, 254 insertions(+), 65 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index 2fc78fc619cab..1d2ec6abc0dd1 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -425,6 +425,22 @@ describe('ElasticIndex', () => { type: 'tsvb-validation-telemetry', }, }, + { + bool: { + must: [ + { + match: { + type: 'search-session', + }, + }, + { + match: { + 'search-session.persisted': false, + }, + }, + ], + }, + }, ], }, }, diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index 462425ff6e3e0..460aabbc77415 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -29,6 +29,46 @@ export interface FullIndexInfo { mappings: IndexMapping; } +// When migrating from the outdated index we use a read query which excludes +// saved objects which are no longer used. These saved objects will still be +// kept in the outdated index for backup purposes, but won't be availble in +// the upgraded index. +export const excludeUnusedTypesQuery: estypes.QueryContainer = { + bool: { + must_not: [ + // https://github.com/elastic/kibana/issues/91869 + { + term: { + type: 'fleet-agent-events', + }, + }, + // https://github.com/elastic/kibana/issues/95617 + { + term: { + type: 'tsvb-validation-telemetry', + }, + }, + // https://github.com/elastic/kibana/issues/96131 + { + bool: { + must: [ + { + match: { + type: 'search-session', + }, + }, + { + match: { + 'search-session.persisted': false, + }, + }, + ], + }, + }, + ], + }, +}; + /** * A slight enhancement to indices.get, that adds indexName, and validates that the * index mappings are somewhat what we expect. @@ -69,23 +109,6 @@ export function reader( const scroll = scrollDuration; let scrollId: string | undefined; - // When migrating from the outdated index we use a read query which excludes - // saved object types which are no longer used. These saved objects will - // still be kept in the outdated index for backup purposes, but won't be - // availble in the upgraded index. - const EXCLUDE_UNUSED_TYPES = [ - 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 - 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 - ]; - - const excludeUnusedTypesQuery = { - bool: { - must_not: EXCLUDE_UNUSED_TYPES.map((type) => ({ - term: { type }, - })), - }, - }; - const nextBatch = () => scrollId !== undefined ? client.scroll>({ diff --git a/src/core/server/saved_objects/migrations/core/index.ts b/src/core/server/saved_objects/migrations/core/index.ts index 322150e2b850e..1e51983a0ffbd 100644 --- a/src/core/server/saved_objects/migrations/core/index.ts +++ b/src/core/server/saved_objects/migrations/core/index.ts @@ -14,3 +14,4 @@ export type { LogFn, SavedObjectsMigrationLogger } from './migration_logger'; export type { MigrationResult, MigrationStatus } from './migration_coordinator'; export { createMigrationEsClient } from './migration_es_client'; export type { MigrationEsClient } from './migration_es_client'; +export { excludeUnusedTypesQuery } from './elastic_index'; diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index 9d6afbd3b0d87..02d3f8e21a510 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -14,7 +14,6 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; import type { ElasticsearchClientError, ResponseError } from '@elastic/elasticsearch/lib/errors'; import { pipe } from 'fp-ts/lib/pipeable'; import { flow } from 'fp-ts/lib/function'; -import { QueryContainer } from '@elastic/eui/src/components/search_bar/query/ast_to_es_query_dsl'; import { ElasticsearchClient } from '../../../elasticsearch'; import { IndexMapping } from '../../mappings'; import { SavedObjectsRawDoc, SavedObjectsRawDocSource } from '../../serialization'; @@ -440,9 +439,9 @@ export const reindex = ( requireAlias: boolean, /* When reindexing we use a source query to exclude saved objects types which * are no longer used. These saved objects will still be kept in the outdated - * index for backup purposes, but won't be availble in the upgraded index. + * index for backup purposes, but won't be available in the upgraded index. */ - unusedTypesToExclude: Option.Option + unusedTypesQuery: Option.Option ): TaskEither.TaskEither => () => { return client .reindex({ @@ -457,14 +456,10 @@ export const reindex = ( // Set reindex batch size size: BATCH_SIZE, // Exclude saved object types - query: Option.fold( + query: Option.fold( () => undefined, - (types) => ({ - bool: { - must_not: types.map((type) => ({ term: { type } })), - }, - }) - )(unusedTypesToExclude), + (query) => query + )(unusedTypesQuery), }, dest: { index: targetIndex, diff --git a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts index 21c05d22b0581..3905044f04e2f 100644 --- a/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts +++ b/src/core/server/saved_objects/migrationsv2/integration_tests/actions.test.ts @@ -416,14 +416,20 @@ describe('migration actions', () => { ] `); }); - it('resolves right and excludes all unusedTypesToExclude documents', async () => { + it('resolves right and excludes all documents not matching the unusedTypesQuery', async () => { const res = (await reindex( client, 'existing_index_with_docs', 'reindex_target_excluded_docs', Option.none, false, - Option.some(['f_agent_event', 'another_unused_type']) + Option.of({ + bool: { + must_not: ['f_agent_event', 'another_unused_type'].map((type) => ({ + term: { type }, + })), + }, + }) )()) as Either.Right; const task = waitForReindexTask(client, res.right.taskId, '10s'); await expect(task()).resolves.toMatchInlineSnapshot(` diff --git a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts index 4d93abcc4018f..fa2e65f16bb2d 100644 --- a/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts +++ b/src/core/server/saved_objects/migrationsv2/migrations_state_action_machine.test.ts @@ -254,12 +254,40 @@ describe('migrationsStateActionMachine', () => { }, }, }, - "unusedTypesToExclude": Object { + "unusedTypesQuery": Object { "_tag": "Some", - "value": Array [ - "fleet-agent-events", - "tsvb-validation-telemetry", - ], + "value": Object { + "bool": Object { + "must_not": Array [ + Object { + "term": Object { + "type": "fleet-agent-events", + }, + }, + Object { + "term": Object { + "type": "tsvb-validation-telemetry", + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "match": Object { + "type": "search-session", + }, + }, + Object { + "match": Object { + "search-session.persisted": false, + }, + }, + ], + }, + }, + ], + }, + }, }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", @@ -322,12 +350,40 @@ describe('migrationsStateActionMachine', () => { }, }, }, - "unusedTypesToExclude": Object { + "unusedTypesQuery": Object { "_tag": "Some", - "value": Array [ - "fleet-agent-events", - "tsvb-validation-telemetry", - ], + "value": Object { + "bool": Object { + "must_not": Array [ + Object { + "term": Object { + "type": "fleet-agent-events", + }, + }, + Object { + "term": Object { + "type": "tsvb-validation-telemetry", + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "match": Object { + "type": "search-session", + }, + }, + Object { + "match": Object { + "search-session.persisted": false, + }, + }, + ], + }, + }, + ], + }, + }, }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", @@ -475,12 +531,40 @@ describe('migrationsStateActionMachine', () => { }, }, }, - "unusedTypesToExclude": Object { + "unusedTypesQuery": Object { "_tag": "Some", - "value": Array [ - "fleet-agent-events", - "tsvb-validation-telemetry", - ], + "value": Object { + "bool": Object { + "must_not": Array [ + Object { + "term": Object { + "type": "fleet-agent-events", + }, + }, + Object { + "term": Object { + "type": "tsvb-validation-telemetry", + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "match": Object { + "type": "search-session", + }, + }, + Object { + "match": Object { + "search-session.persisted": false, + }, + }, + ], + }, + }, + ], + }, + }, }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", @@ -538,12 +622,40 @@ describe('migrationsStateActionMachine', () => { }, }, }, - "unusedTypesToExclude": Object { + "unusedTypesQuery": Object { "_tag": "Some", - "value": Array [ - "fleet-agent-events", - "tsvb-validation-telemetry", - ], + "value": Object { + "bool": Object { + "must_not": Array [ + Object { + "term": Object { + "type": "fleet-agent-events", + }, + }, + Object { + "term": Object { + "type": "tsvb-validation-telemetry", + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "match": Object { + "type": "search-session", + }, + }, + Object { + "match": Object { + "search-session.persisted": false, + }, + }, + ], + }, + }, + ], + }, + }, }, "versionAlias": ".my-so-index_7.11.0", "versionIndex": ".my-so-index_7.11.0_001", diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index 8aad62f13b8fe..0267ae33dd157 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -70,7 +70,17 @@ describe('migrations v2 model', () => { versionAlias: '.kibana_7.11.0', versionIndex: '.kibana_7.11.0_001', tempIndex: '.kibana_7.11.0_reindex_temp', - unusedTypesToExclude: Option.some(['unused-fleet-agent-events']), + unusedTypesQuery: Option.of({ + bool: { + must_not: [ + { + term: { + type: 'unused-fleet-agent-events', + }, + }, + ], + }, + }), }; describe('exponential retry delays for retryable_es_client_error', () => { @@ -1177,12 +1187,40 @@ describe('migrations v2 model', () => { }, }, }, - "unusedTypesToExclude": Object { + "unusedTypesQuery": Object { "_tag": "Some", - "value": Array [ - "fleet-agent-events", - "tsvb-validation-telemetry", - ], + "value": Object { + "bool": Object { + "must_not": Array [ + Object { + "term": Object { + "type": "fleet-agent-events", + }, + }, + Object { + "term": Object { + "type": "tsvb-validation-telemetry", + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "match": Object { + "type": "search-session", + }, + }, + Object { + "match": Object { + "search-session.persisted": false, + }, + }, + ], + }, + }, + ], + }, + }, }, "versionAlias": ".kibana_task_manager_8.1.0", "versionIndex": ".kibana_task_manager_8.1.0_001", diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index ee78692a7044f..acf0f620136a2 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -16,6 +16,7 @@ import { IndexMapping } from '../mappings'; import { ResponseType } from './next'; import { SavedObjectsMigrationVersion } from '../types'; import { disableUnknownTypeMappingFields } from '../migrations/core/migration_context'; +import { excludeUnusedTypesQuery } from '../migrations/core'; import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; /** @@ -74,6 +75,7 @@ function indexBelongsToLaterVersion(indexName: string, kibanaVersion: string): b const version = valid(indexVersion(indexName)); return version != null ? gt(version, kibanaVersion) : false; } + /** * Extracts the version number from a >= 7.11 index * @param indexName A >= v7.11 index name @@ -781,11 +783,6 @@ export const createInitialState = ({ }, }; - const unusedTypesToExclude = Option.some([ - 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 - 'tsvb-validation-telemetry', // https://github.com/elastic/kibana/issues/95617 - ]); - const initialState: InitState = { controlState: 'INIT', indexPrefix, @@ -804,7 +801,7 @@ export const createInitialState = ({ retryAttempts: migrationsConfig.retryAttempts, batchSize: migrationsConfig.batchSize, logs: [], - unusedTypesToExclude, + unusedTypesQuery: Option.of(excludeUnusedTypesQuery), }; return initialState; }; diff --git a/src/core/server/saved_objects/migrationsv2/next.ts b/src/core/server/saved_objects/migrationsv2/next.ts index 5cbda741a0ce5..bb506cbca66fb 100644 --- a/src/core/server/saved_objects/migrationsv2/next.ts +++ b/src/core/server/saved_objects/migrationsv2/next.ts @@ -70,7 +70,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra state.tempIndex, Option.none, false, - state.unusedTypesToExclude + state.unusedTypesQuery ), SET_TEMP_WRITE_BLOCK: (state: SetTempWriteBlock) => Actions.setWriteBlock(client, state.tempIndex), @@ -115,7 +115,7 @@ export const nextActionMap = (client: ElasticsearchClient, transformRawDocs: Tra state.sourceIndex.value, state.preMigrationScript, false, - state.unusedTypesToExclude + state.unusedTypesQuery ), LEGACY_REINDEX_WAIT_FOR_TASK: (state: LegacyReindexWaitForTaskState) => Actions.waitForReindexTask(client, state.legacyReindexTaskId, '60s'), diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index e9b351c0152fc..5e84bc23b1d16 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -7,6 +7,7 @@ */ import * as Option from 'fp-ts/lib/Option'; +import { estypes } from '@elastic/elasticsearch'; import { ControlState } from './state_action_machine'; import { AliasAction } from './actions'; import { IndexMapping } from '../mappings'; @@ -91,9 +92,9 @@ export interface BaseState extends ControlState { readonly tempIndex: string; /* When reindexing we use a source query to exclude saved objects types which * are no longer used. These saved objects will still be kept in the outdated - * index for backup purposes, but won't be availble in the upgraded index. + * index for backup purposes, but won't be available in the upgraded index. */ - readonly unusedTypesToExclude: Option.Option; + readonly unusedTypesQuery: Option.Option; } export type InitState = BaseState & {