From 24064a477b78ba372c015769a4cc950896c29bc2 Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Tue, 25 Apr 2023 18:48:30 +0200 Subject: [PATCH 1/2] Make indices.getMapping() resilient to intermitent failures --- .../src/kibana_migrator_utils.test.ts | 140 ++++++++++++++++++ .../src/kibana_migrator_utils.ts | 14 +- 2 files changed, 151 insertions(+), 3 deletions(-) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts index 8927b0e82f8aa..0698904c45d30 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.test.ts @@ -7,13 +7,16 @@ */ import { errors } from '@elastic/elasticsearch'; +import type { IndicesGetMappingResponse } from '@elastic/elasticsearch/lib/api/types'; import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; import { MAIN_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; import { loggerMock } from '@kbn/logging-mocks'; import { DEFAULT_INDEX_TYPES_MAP } from './kibana_migrator_constants'; import { calculateTypeStatuses, createMultiPromiseDefer, + getCurrentIndexTypesMap, getIndicesInvolvedInRelocation, indexMapToIndexTypesMap, } from './kibana_migrator_utils'; @@ -41,6 +44,143 @@ describe('createMultiPromiseDefer', () => { }); }); +describe('getCurrentIndexTypesMap', () => { + const defaultIndexTypesMap: IndexTypesMap = { + '.my_index': ['type1', 'type2', 'type3'], + '.task_index': ['type4'], + }; + + describe('when mainIndex does NOT exist', () => { + it('assumes we are targeting a fresh ES cluster', async () => { + const notFoundError = new errors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 404, + body: { ok: false, message: 'Unknown resource.' }, + }) + ); + const client = elasticsearchClientMock.createInternalClient( + elasticsearchClientMock.createErrorTransportRequestPromise(notFoundError) + ); + + const currentIndexTypesMap = await getCurrentIndexTypesMap({ + client, + mainIndex: '.my_index', + defaultIndexTypesMap: {}, + logger: loggerMock.create(), + }); + + expect(currentIndexTypesMap).toBeUndefined(); + }); + }); + + describe('when mainIndex exists, but it does not have an indexTypesMap in its mapping._meta', () => { + it('returns the defaultIndexTypesMap', async () => { + const getMappingResponse: IndicesGetMappingResponse = { + '.my_index_8.8.0_001': { + mappings: { + _meta: { + migrationMappingPropertyHashes: { + application_usage_daily: '43b8830d5d0df85a6823d290885fc9fd', + application_usage_totals: '3d1b76c39bfb2cc8296b024d73854724', + application_usage_transactional: '3d1b76c39bfb2cc8296b024d73854724', + }, + }, + }, + }, + }; + + const client = elasticsearchClientMock.createInternalClient(); + client.indices.getMapping.mockResolvedValueOnce(getMappingResponse); + + const currentIndexTypesMap = await getCurrentIndexTypesMap({ + client, + mainIndex: '.my_index', + defaultIndexTypesMap, + logger: loggerMock.create(), + }); + + expect(currentIndexTypesMap).toEqual(defaultIndexTypesMap); + }); + }); + + describe('when mainIndex exists and it does have an indexTypesMap property in its mapping._meta', () => { + it('returns the stored indexTypesMap', async () => { + const storedIndexTypesMap: IndexTypesMap = { + '.my_index': ['type1', 'type2'], + '.task_index': ['type4'], + '.other_index': ['type3'], + }; + const getMappingResponse: IndicesGetMappingResponse = { + '.my_index_8.8.0_001': { + mappings: { + _meta: { + migrationMappingPropertyHashes: { + application_usage_daily: '43b8830d5d0df85a6823d290885fc9fd', + application_usage_totals: '3d1b76c39bfb2cc8296b024d73854724', + application_usage_transactional: '3d1b76c39bfb2cc8296b024d73854724', + }, + indexTypesMap: storedIndexTypesMap, + }, + }, + }, + }; + + const client = elasticsearchClientMock.createInternalClient(); + client.indices.getMapping.mockResolvedValueOnce(getMappingResponse); + + const currentIndexTypesMap = await getCurrentIndexTypesMap({ + client, + mainIndex: '.my_index', + defaultIndexTypesMap, + logger: loggerMock.create(), + }); + + expect(currentIndexTypesMap).toEqual(storedIndexTypesMap); + }); + }); + + describe('when retriable errors occur', () => { + it('keeps trying to fetch the index mappings forever', async () => { + const unavailable = new errors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 503, + headers: { 'Retry-After': '30' }, + body: 'Kibana server is not ready yet', + }) + ); + + const notFound = new errors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 404, + body: { ok: false, message: 'Unknown resource.' }, + }) + ); + + const client = elasticsearchClientMock.createInternalClient(); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(unavailable); + client.indices.getMapping.mockRejectedValueOnce(notFound); + + await getCurrentIndexTypesMap({ + client, + mainIndex: '.my_index', + defaultIndexTypesMap, + logger: loggerMock.create(), + retryDelay: 1, + }); + + expect(client.indices.getMapping).toHaveBeenCalledTimes(10); + }); + }); +}); + describe('getIndicesInvolvedInRelocation', () => { const getIndicesInvolvedInRelocationParams = () => { const client = elasticsearchClientMock.createElasticsearchClient(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts index 408e88721c413..36d17cd1159e9 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator_utils.ts @@ -7,6 +7,7 @@ */ import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { migrationRetryCallCluster } from '@kbn/core-elasticsearch-server-internal'; import type { IndexTypesMap } from '@kbn/core-saved-objects-base-server-internal'; import type { Logger } from '@kbn/logging'; import type { IndexMap } from './core'; @@ -40,17 +41,24 @@ export async function getCurrentIndexTypesMap({ mainIndex, defaultIndexTypesMap, logger, + retryDelay = 2500, }: { client: ElasticsearchClient; mainIndex: string; defaultIndexTypesMap: IndexTypesMap; logger: Logger; + retryDelay?: number; }): Promise { try { // check if the main index (i.e. .kibana) exists - const mapping = await client.indices.getMapping({ - index: mainIndex, - }); + const mapping = await migrationRetryCallCluster( + () => + client.indices.getMapping({ + index: mainIndex, + }), + logger, + retryDelay + ); // main index exists, try to extract the indexTypesMap from _meta const meta = Object.values(mapping)?.[0]?.mappings._meta; From 3b58e4412dd005369d69513ff85d5b6a6e6a2ee2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 25 Apr 2023 16:58:49 +0000 Subject: [PATCH 2/2] [CI] Auto-commit changed files from 'node scripts/lint_ts_projects --fix' --- .../core-saved-objects-migration-server-internal/tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json index a770476aaf392..d21138e25db84 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/tsconfig.json @@ -30,6 +30,7 @@ "@kbn/safer-lodash-set", "@kbn/logging-mocks", "@kbn/core-saved-objects-base-server-mocks", + "@kbn/core-elasticsearch-server-internal", ], "exclude": [ "target/**/*",