From 30ee115c1f487e6224ac2df3a9660197b3a66cd4 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 20 Apr 2022 16:41:18 +0100 Subject: [PATCH 1/6] remove legacy component templates as part of package install --- .../template/remove_legacy.test.ts | 98 +++++++++++++++++++ .../elasticsearch/template/remove_legacy.ts | 89 +++++++++++++++++ .../services/epm/packages/_install_package.ts | 7 ++ 3 files changed, 194 insertions(+) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts new file mode 100644 index 0000000000000..d3e50fafa9dd7 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; + +import uuid from 'uuid'; + +import type { InstallablePackage, RegistryDataStream } from '../../../../types'; + +import { _getLegacyComponentTemplatesForPackage } from './remove_legacy'; + +const pickRandom = (arr: any[]) => arr[Math.floor(Math.random() * arr.length)]; +const pickRandomType = pickRandom.bind(null, ['logs', 'metrics']); +const createMockDataStream = ({ + packageName = 'somepkg', + dataset, +}: { + packageName?: string; + dataset?: string; +} = {}) => { + return { + type: pickRandomType(), + dataset: dataset || uuid.v4(), + title: packageName, + package: packageName, + path: 'some_path', + release: 'ga', + } as RegistryDataStream; +}; +const createMockTemplate = ({ + name = 'templateName', + packageName = 'somePackage', +}: { + name?: string; + packageName?: string; +} = {}) => { + return { + name, + component_template: { + _meta: { + package: packageName, + }, + template: { + settings: {}, + }, + }, + } as ClusterComponentTemplate; +}; + +const makeArrayOf = (arraySize: number, fn = (i: any) => i) => { + return [...Array(arraySize)].map(fn); +}; +describe('_getLegacyComponentTemplatesForPackage', () => { + it('should handle empty array', () => { + const templates = [] as ClusterComponentTemplate[]; + const pkg = {} as InstallablePackage; + + const result = _getLegacyComponentTemplatesForPackage(templates, pkg); + expect(result).toEqual([]); + }); + it('should return empty array if no legacy templates', () => { + const templates = makeArrayOf(1000, createMockTemplate); + const pkg = { + data_streams: makeArrayOf(100, createMockDataStream), + } as InstallablePackage; + + const result = _getLegacyComponentTemplatesForPackage(templates, pkg); + expect(result).toEqual([]); + }); + + it('should find legacy templates', () => { + const packageName = 'myPkg'; + const legacyTemplates = [ + 'logs-mypkg.dataset@settings', + 'logs-mypkg.dataset@mappings', + 'logs-mypkg.dataset2@mappings', + 'logs-mypkg.dataset2@settings', + ]; + const templates = [ + ...makeArrayOf(100, createMockTemplate), + ...legacyTemplates.map((name) => createMockTemplate({ name, packageName })), + ]; + const pkg = { + data_streams: [ + ...makeArrayOf(20, createMockDataStream), + createMockDataStream({ packageName, dataset: 'mypkg.dataset' }), + createMockDataStream({ packageName, dataset: 'mypkg.dataset2' }), + ], + } as InstallablePackage; + + const result = _getLegacyComponentTemplatesForPackage(templates, pkg); + expect(result).toEqual(legacyTemplates); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts new file mode 100644 index 0000000000000..e4114dba1425f --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; + +import type { InstallablePackage, RegistryDataStream } from '../../../../types'; +import { getRegistryDataStreamAssetBaseName } from '..'; +const LEGACY_TEMPLATE_SUFFIXES = ['@mappings', '@settings']; + +const getComponentTemplateWithSuffix = (dataStream: RegistryDataStream, suffix: string) => { + const baseName = getRegistryDataStreamAssetBaseName(dataStream); + + return baseName + suffix; +}; + +export const _getLegacyComponentTemplatesForPackage = ( + componentTemplates: ClusterComponentTemplate[], + installablePackage: InstallablePackage +): string[] => { + const namesMap: Map = new Map(); + + // fill a map with all possible @mappings and @settings component + // template names for fast lookup below. + installablePackage.data_streams?.forEach((ds) => { + LEGACY_TEMPLATE_SUFFIXES.forEach((suffix) => { + namesMap.set(getComponentTemplateWithSuffix(ds, suffix)); + }); + }); + + return componentTemplates.reduce((legacyTemplates, componentTemplate) => { + if (!namesMap.has(componentTemplate.name)) return legacyTemplates; + + if (componentTemplate.component_template._meta?.package?.name !== installablePackage.name) + return legacyTemplates; + + return legacyTemplates.concat(componentTemplate.name); + }, []); +}; + +const _deleteComponentTemplates = async (params: { + templateNames: string[]; + esClient: ElasticsearchClient; + logger: Logger; +}): Promise => { + const { templateNames, esClient, logger } = params; + const deleteResults = await Promise.allSettled( + templateNames.map((name) => esClient.cluster.deleteComponentTemplate({ name })) + ); + + const errors = deleteResults.filter((r) => r.status === 'rejected') as PromiseRejectedResult[]; + + if (errors.length) { + const prettyErrors = errors.map((e) => `"${e.reason}"`).join(', '); + logger.debug( + `Encountered ${errors.length} errors deleting legacy component templates: ${prettyErrors}` + ); + } +}; + +const _getAllComponentTemplates = async (esClient: ElasticsearchClient) => + esClient.cluster.getComponentTemplate().then((result) => result.component_templates); + +export const removeLegacyTemplates = async (params: { + packageInfo: InstallablePackage; + esClient: ElasticsearchClient; + logger: Logger; +}): Promise => { + const { packageInfo, esClient, logger } = params; + + const allComponentTemplates = await _getAllComponentTemplates(esClient); + + const legacyComponentTemplateNames = await _getLegacyComponentTemplatesForPackage( + allComponentTemplates, + packageInfo + ); + + if (!legacyComponentTemplateNames.length) return; + + await _deleteComponentTemplates({ + templateNames: legacyComponentTemplateNames, + esClient, + logger, + }); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index b4e673b8a9da4..34ada19685f83 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -23,6 +23,7 @@ import type { InstallablePackage, InstallSource, PackageAssetReference } from '. import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; import { installTemplates } from '../elasticsearch/template/install'; +import { removeLegacyTemplates } from '../elasticsearch/template/remove_legacy'; import { installPipelines, isTopLevelPipeline, @@ -161,6 +162,12 @@ export async function _installPackage({ savedObjectsClient ); + try { + await removeLegacyTemplates({ packageInfo, esClient, logger }); + } catch (e) { + logger.warn(`Error removing legacy templates: ${e.message}`); + } + // update current backing indices of each data stream await updateCurrentWriteIndices(esClient, logger, installedTemplates); From 2b4665d6299aef0e6655e624676bc33ff1654adc Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 20 Apr 2022 17:12:25 +0100 Subject: [PATCH 2/6] re-work unit tests --- .../template/remove_legacy.test.ts | 72 +++++++++++++------ 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts index d3e50fafa9dd7..b4e5b6d623536 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts @@ -16,14 +16,16 @@ import { _getLegacyComponentTemplatesForPackage } from './remove_legacy'; const pickRandom = (arr: any[]) => arr[Math.floor(Math.random() * arr.length)]; const pickRandomType = pickRandom.bind(null, ['logs', 'metrics']); const createMockDataStream = ({ - packageName = 'somepkg', + packageName, + type, dataset, }: { - packageName?: string; + packageName: string; + type?: string; dataset?: string; -} = {}) => { +}) => { return { - type: pickRandomType(), + type: type || pickRandomType(), dataset: dataset || uuid.v4(), title: packageName, package: packageName, @@ -33,16 +35,16 @@ const createMockDataStream = ({ }; const createMockTemplate = ({ name = 'templateName', - packageName = 'somePackage', + packageName, }: { name?: string; - packageName?: string; -} = {}) => { + packageName: string; +}) => { return { name, component_template: { _meta: { - package: packageName, + package: { name: packageName }, }, template: { settings: {}, @@ -55,17 +57,19 @@ const makeArrayOf = (arraySize: number, fn = (i: any) => i) => { return [...Array(arraySize)].map(fn); }; describe('_getLegacyComponentTemplatesForPackage', () => { - it('should handle empty array', () => { + it('should handle empty templates array', () => { const templates = [] as ClusterComponentTemplate[]; - const pkg = {} as InstallablePackage; + const pkg = { name: 'testPkg', data_streams: [] as RegistryDataStream[] } as InstallablePackage; const result = _getLegacyComponentTemplatesForPackage(templates, pkg); expect(result).toEqual([]); }); it('should return empty array if no legacy templates', () => { - const templates = makeArrayOf(1000, createMockTemplate); + const packageName = 'testPkg'; + const templates = makeArrayOf(1000, () => createMockTemplate({ packageName })); const pkg = { - data_streams: makeArrayOf(100, createMockDataStream), + name: packageName, + data_streams: makeArrayOf(100, () => createMockDataStream({ packageName })), } as InstallablePackage; const result = _getLegacyComponentTemplatesForPackage(templates, pkg); @@ -73,26 +77,52 @@ describe('_getLegacyComponentTemplatesForPackage', () => { }); it('should find legacy templates', () => { - const packageName = 'myPkg'; + const packageName = 'testPkg'; const legacyTemplates = [ - 'logs-mypkg.dataset@settings', - 'logs-mypkg.dataset@mappings', - 'logs-mypkg.dataset2@mappings', - 'logs-mypkg.dataset2@settings', + 'logs-testPkg.dataset@settings', + 'logs-testPkg.dataset@mappings', + 'metrics-testPkg.dataset2@mappings', + 'metrics-testPkg.dataset2@settings', ]; const templates = [ - ...makeArrayOf(100, createMockTemplate), + ...makeArrayOf(100, () => createMockTemplate({ packageName })), ...legacyTemplates.map((name) => createMockTemplate({ name, packageName })), ]; const pkg = { + name: packageName, data_streams: [ - ...makeArrayOf(20, createMockDataStream), - createMockDataStream({ packageName, dataset: 'mypkg.dataset' }), - createMockDataStream({ packageName, dataset: 'mypkg.dataset2' }), + ...makeArrayOf(20, () => createMockDataStream({ packageName })), + createMockDataStream({ type: 'logs', packageName, dataset: 'testPkg.dataset' }), + createMockDataStream({ type: 'metrics', packageName, dataset: 'testPkg.dataset2' }), ], } as InstallablePackage; const result = _getLegacyComponentTemplatesForPackage(templates, pkg); expect(result).toEqual(legacyTemplates); }); + + it('should only return templates if package name matches as well', () => { + const packageName = 'testPkg'; + const legacyTemplates = [ + 'logs-testPkg.dataset@settings', + 'logs-testPkg.dataset@mappings', + 'metrics-testPkg.dataset2@mappings', + 'metrics-testPkg.dataset2@settings', + ]; + const templates = [ + ...makeArrayOf(20, () => createMockTemplate({ packageName })), + ...legacyTemplates.map((name) => createMockTemplate({ name, packageName: 'someOtherPkg' })), + ]; + const pkg = { + name: packageName, + data_streams: [ + ...makeArrayOf(20, () => createMockDataStream({ packageName })), + createMockDataStream({ type: 'logs', packageName, dataset: 'testPkg.dataset' }), + createMockDataStream({ type: 'metrics', packageName, dataset: 'testPkg.dataset2' }), + ], + } as InstallablePackage; + + const result = _getLegacyComponentTemplatesForPackage(templates, pkg); + expect(result).toEqual([]); + }); }); From 7823d63f22fe448751020f41463e359fd6d91dbb Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 20 Apr 2022 17:15:50 +0100 Subject: [PATCH 3/6] remove unnecessary await --- .../server/services/epm/elasticsearch/template/remove_legacy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts index e4114dba1425f..a5abd31021dda 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts @@ -74,7 +74,7 @@ export const removeLegacyTemplates = async (params: { const allComponentTemplates = await _getAllComponentTemplates(esClient); - const legacyComponentTemplateNames = await _getLegacyComponentTemplatesForPackage( + const legacyComponentTemplateNames = _getLegacyComponentTemplatesForPackage( allComponentTemplates, packageInfo ); From 74e54e497e7af3ca6ea79767245eb71bc8c9a100 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 20 Apr 2022 21:01:40 +0100 Subject: [PATCH 4/6] check if component templates are in use before deleting --- .../template/remove_legacy.test.ts | 150 +++++++++++++++++- .../elasticsearch/template/remove_legacy.ts | 74 ++++++++- 2 files changed, 214 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts index b4e5b6d623536..4877d28cbead0 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.test.ts @@ -5,13 +5,26 @@ * 2.0. */ -import type { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; +import type { + ClusterComponentTemplate, + IndicesGetIndexTemplateIndexTemplateItem, +} from '@elastic/elasticsearch/lib/api/types'; + +import type { Logger } from '@kbn/core/server'; import uuid from 'uuid'; +import { loggingSystemMock } from '@kbn/core/server/mocks'; + import type { InstallablePackage, RegistryDataStream } from '../../../../types'; -import { _getLegacyComponentTemplatesForPackage } from './remove_legacy'; +import { + _getLegacyComponentTemplatesForPackage, + _getIndexTemplatesToUsedByMap, + _filterComponentTemplatesInUse, +} from './remove_legacy'; + +const mockLogger: Logger = loggingSystemMock.create().get(); const pickRandom = (arr: any[]) => arr[Math.floor(Math.random() * arr.length)]; const pickRandomType = pickRandom.bind(null, ['logs', 'metrics']); @@ -33,7 +46,7 @@ const createMockDataStream = ({ release: 'ga', } as RegistryDataStream; }; -const createMockTemplate = ({ +const createMockComponentTemplate = ({ name = 'templateName', packageName, }: { @@ -53,6 +66,14 @@ const createMockTemplate = ({ } as ClusterComponentTemplate; }; +const createMockTemplate = ({ name, composedOf = [] }: { name: string; composedOf?: string[] }) => + ({ + name, + index_template: { + composed_of: composedOf, + }, + } as IndicesGetIndexTemplateIndexTemplateItem); + const makeArrayOf = (arraySize: number, fn = (i: any) => i) => { return [...Array(arraySize)].map(fn); }; @@ -66,7 +87,7 @@ describe('_getLegacyComponentTemplatesForPackage', () => { }); it('should return empty array if no legacy templates', () => { const packageName = 'testPkg'; - const templates = makeArrayOf(1000, () => createMockTemplate({ packageName })); + const templates = makeArrayOf(1000, () => createMockComponentTemplate({ packageName })); const pkg = { name: packageName, data_streams: makeArrayOf(100, () => createMockDataStream({ packageName })), @@ -85,8 +106,8 @@ describe('_getLegacyComponentTemplatesForPackage', () => { 'metrics-testPkg.dataset2@settings', ]; const templates = [ - ...makeArrayOf(100, () => createMockTemplate({ packageName })), - ...legacyTemplates.map((name) => createMockTemplate({ name, packageName })), + ...makeArrayOf(100, () => createMockComponentTemplate({ packageName })), + ...legacyTemplates.map((name) => createMockComponentTemplate({ name, packageName })), ]; const pkg = { name: packageName, @@ -110,8 +131,10 @@ describe('_getLegacyComponentTemplatesForPackage', () => { 'metrics-testPkg.dataset2@settings', ]; const templates = [ - ...makeArrayOf(20, () => createMockTemplate({ packageName })), - ...legacyTemplates.map((name) => createMockTemplate({ name, packageName: 'someOtherPkg' })), + ...makeArrayOf(20, () => createMockComponentTemplate({ packageName })), + ...legacyTemplates.map((name) => + createMockComponentTemplate({ name, packageName: 'someOtherPkg' }) + ), ]; const pkg = { name: packageName, @@ -126,3 +149,114 @@ describe('_getLegacyComponentTemplatesForPackage', () => { expect(result).toEqual([]); }); }); + +describe('_getIndexTemplatesToUsedByMap', () => { + it('should return empty map if no index templates provided', () => { + const indexTemplates = [] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _getIndexTemplatesToUsedByMap(indexTemplates); + + expect(result.size).toEqual(0); + }); + + it('should return empty map if no index templates have no component templates', () => { + const indexTemplates = [createMockTemplate({ name: 'tmpl1' })]; + + const result = _getIndexTemplatesToUsedByMap(indexTemplates); + + expect(result.size).toEqual(0); + }); + + it('should return correct map if templates have composedOf', () => { + const indexTemplates = [ + createMockTemplate({ name: 'tmpl1' }), + createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp1'] }), + createMockTemplate({ name: 'tmpl3', composedOf: ['ctmp1', 'ctmp2'] }), + createMockTemplate({ name: 'tmpl4', composedOf: ['ctmp3'] }), + ]; + + const expectedMap = { + ctmp1: ['tmpl2', 'tmpl3'], + ctmp2: ['tmpl3'], + ctmp3: ['tmpl4'], + }; + + const result = _getIndexTemplatesToUsedByMap(indexTemplates); + + expect(Object.fromEntries(result)).toEqual(expectedMap); + }); +}); + +describe('_filterComponentTemplatesInUse', () => { + it('should return empty array if provided with empty component templates', () => { + const componentTemplateNames = [] as string[]; + const indexTemplates = [] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _filterComponentTemplatesInUse({ + componentTemplateNames, + indexTemplates, + logger: mockLogger, + }); + + expect(result).toHaveLength(0); + }); + + it('should remove component template used by index template ', () => { + const componentTemplateNames = ['ctmp1', 'ctmp2'] as string[]; + const indexTemplates = [ + createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1'] }), + ] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _filterComponentTemplatesInUse({ + componentTemplateNames, + indexTemplates, + logger: mockLogger, + }); + + expect(result).toEqual(['ctmp2']); + }); + it('should remove component templates used by one index template ', () => { + const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[]; + const indexTemplates = [ + createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1', 'ctmp2'] }), + ] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _filterComponentTemplatesInUse({ + componentTemplateNames, + indexTemplates, + logger: mockLogger, + }); + + expect(result).toEqual(['ctmp3']); + }); + it('should remove component templates used by different index templates ', () => { + const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[]; + const indexTemplates = [ + createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1'] }), + createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp2'] }), + ] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _filterComponentTemplatesInUse({ + componentTemplateNames, + indexTemplates, + logger: mockLogger, + }); + + expect(result).toEqual(['ctmp3']); + }); + it('should remove component templates used by multiple index templates ', () => { + const componentTemplateNames = ['ctmp1', 'ctmp2', 'ctmp3'] as string[]; + const indexTemplates = [ + createMockTemplate({ name: 'tmpl1', composedOf: ['ctmp1', 'ctmp2'] }), + createMockTemplate({ name: 'tmpl2', composedOf: ['ctmp2', 'ctmp1'] }), + ] as IndicesGetIndexTemplateIndexTemplateItem[]; + + const result = _filterComponentTemplatesInUse({ + componentTemplateNames, + indexTemplates, + logger: mockLogger, + }); + + expect(result).toEqual(['ctmp3']); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts index a5abd31021dda..a2f9586a7f08c 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { ClusterComponentTemplate } from '@elastic/elasticsearch/lib/api/types'; +import type { + ClusterComponentTemplate, + IndicesGetIndexTemplateIndexTemplateItem, +} from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { InstallablePackage, RegistryDataStream } from '../../../../types'; @@ -62,9 +65,61 @@ const _deleteComponentTemplates = async (params: { } }; +export const _getIndexTemplatesToUsedByMap = ( + indexTemplates: IndicesGetIndexTemplateIndexTemplateItem[] +) => { + const lookupMap: Map = new Map(); + + indexTemplates.forEach(({ name: indexTemplateName, index_template: indexTemplate }) => { + const composedOf = indexTemplate?.composed_of; + + if (!composedOf) return; + + composedOf.forEach((componentTemplateName) => { + const existingEntry = lookupMap.get(componentTemplateName) || []; + + lookupMap.set(componentTemplateName, existingEntry.concat(indexTemplateName)); + }); + }); + return lookupMap; +}; + const _getAllComponentTemplates = async (esClient: ElasticsearchClient) => esClient.cluster.getComponentTemplate().then((result) => result.component_templates); +const _getAllIndexTemplatesWithComposedOf = async (esClient: ElasticsearchClient) => + esClient.indices + .getIndexTemplate() + .then((result) => + result.index_templates.filter((tmpl) => tmpl.index_template.composed_of?.length) + ); + +export const _filterComponentTemplatesInUse = ({ + componentTemplateNames, + indexTemplates, + logger, +}: { + componentTemplateNames: string[]; + indexTemplates: IndicesGetIndexTemplateIndexTemplateItem[]; + logger: Logger; +}): string[] => { + const usedByLookup = _getIndexTemplatesToUsedByMap(indexTemplates); + + return componentTemplateNames.filter((componentTemplateName) => { + const indexTemplatesUsingComponentTemplate = usedByLookup.get(componentTemplateName); + + if (indexTemplatesUsingComponentTemplate?.length) { + const prettyTemplates = indexTemplatesUsingComponentTemplate.join(', '); + logger.debug( + `Not deleting legacy template ${componentTemplateName} as it is in use by index templates: ${prettyTemplates}` + ); + return false; + } + + return true; + }); +}; + export const removeLegacyTemplates = async (params: { packageInfo: InstallablePackage; esClient: ElasticsearchClient; @@ -81,8 +136,23 @@ export const removeLegacyTemplates = async (params: { if (!legacyComponentTemplateNames.length) return; + // all index templates that are composed of at least one component template + const allIndexTemplatesWithComposedOf = await _getAllIndexTemplatesWithComposedOf(esClient); + + let templatesToDelete = legacyComponentTemplateNames; + if (allIndexTemplatesWithComposedOf.length) { + // get the component templates not in use by any index templates + templatesToDelete = _filterComponentTemplatesInUse({ + componentTemplateNames: legacyComponentTemplateNames, + indexTemplates: allIndexTemplatesWithComposedOf, + logger, + }); + } + + if (!templatesToDelete.length) return; + await _deleteComponentTemplates({ - templateNames: legacyComponentTemplateNames, + templateNames: templatesToDelete, esClient, logger, }); From b453be087bc6cfcc1333bf337376cc2249f30080 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Wed, 20 Apr 2022 23:11:55 +0100 Subject: [PATCH 5/6] add integration tests --- .../fleet_api_integration/apis/epm/index.js | 1 + .../apis/epm/remove_legacy_templates.ts | 152 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 x-pack/test/fleet_api_integration/apis/epm/remove_legacy_templates.ts diff --git a/x-pack/test/fleet_api_integration/apis/epm/index.js b/x-pack/test/fleet_api_integration/apis/epm/index.js index 6364b1de59fc8..ef103592dfb45 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/index.js +++ b/x-pack/test/fleet_api_integration/apis/epm/index.js @@ -27,6 +27,7 @@ export default function loadTests({ loadTestFile }) { loadTestFile(require.resolve('./update_assets')); loadTestFile(require.resolve('./data_stream')); loadTestFile(require.resolve('./package_install_complete')); + loadTestFile(require.resolve('./remove_legacy_templates')); loadTestFile(require.resolve('./install_error_rollback')); loadTestFile(require.resolve('./final_pipeline')); }); diff --git a/x-pack/test/fleet_api_integration/apis/epm/remove_legacy_templates.ts b/x-pack/test/fleet_api_integration/apis/epm/remove_legacy_templates.ts new file mode 100644 index 0000000000000..53022461244b3 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/epm/remove_legacy_templates.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import path from 'path'; +import fs from 'fs'; +import { promisify } from 'util'; +import { FtrProviderContext } from '../../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../../helpers'; +import { setupFleetAndAgents } from '../agents/services'; +const sleep = promisify(setTimeout); + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const dockerServers = getService('dockerServers'); + const server = dockerServers.get('registry'); + const esClient = getService('es'); + + const uploadPkgName = 'apache'; + const uploadPkgVersion = '0.1.4'; + + const installUploadPackage = async () => { + const buf = fs.readFileSync(testPkgArchiveZip); + await supertest + .post(`/api/fleet/epm/packages`) + .set('kbn-xsrf', 'xxxx') + .type('application/zip') + .send(buf) + .expect(200); + }; + + const testPkgArchiveZip = path.join( + path.dirname(__filename), + '../fixtures/direct_upload_packages/apache_0.1.4.zip' + ); + + const legacyComponentTemplates = [ + { + name: 'logs-apache.access@settings', + template: { + settings: { + index: { + lifecycle: { + name: 'idontexist', + }, + }, + }, + }, + _meta: { + package: { + name: 'apache', + }, + }, + }, + { + name: 'logs-apache.access@mappings', + template: { + mappings: { + dynamic: false, + }, + }, + _meta: { + package: { + name: 'apache', + }, + }, + }, + ]; + const createLegacyComponentTemplates = async () => + Promise.all( + legacyComponentTemplates.map((tmpl) => esClient.cluster.putComponentTemplate(tmpl)) + ); + + const deleteLegacyComponentTemplates = async () => { + esClient.cluster + .deleteComponentTemplate({ name: legacyComponentTemplates.map((t) => t.name) }) + .catch((e) => {}); + }; + + const waitUntilLegacyComponentTemplatesCreated = async () => { + const legacyTemplateNames = legacyComponentTemplates.map((t) => t.name); + for (let i = 5; i > 0; i--) { + const { component_templates: ctmps } = await esClient.cluster.getComponentTemplate(); + + const createdTemplates = ctmps.filter((tmp) => legacyTemplateNames.includes(tmp.name)); + + if (createdTemplates.length === legacyTemplateNames.length) return; + + await sleep(500); + } + + throw new Error('Legacy component templates not created after 5 attempts'); + }; + const uninstallPackage = async (pkg: string, version: string) => { + await supertest.delete(`/api/fleet/epm/packages/${pkg}/${version}`).set('kbn-xsrf', 'xxxx'); + }; + + describe('legacy component template removal', async () => { + skipIfNoDockerRegistry(providerContext); + setupFleetAndAgents(providerContext); + + afterEach(async () => { + if (!server.enabled) return; + await deleteLegacyComponentTemplates(); + await uninstallPackage(uploadPkgName, uploadPkgVersion); + }); + + after(async () => { + await esClient.indices.deleteIndexTemplate({ name: 'testtemplate' }); + }); + + it('should remove legacy component templates if not in use by index templates', async () => { + await createLegacyComponentTemplates(); + + await waitUntilLegacyComponentTemplatesCreated(); + await installUploadPackage(); + + const { component_templates: allComponentTemplates } = + await esClient.cluster.getComponentTemplate(); + const allComponentTemplateNames = allComponentTemplates.map((t) => t.name); + + expect(allComponentTemplateNames.includes('logs-apache.access@settings')).to.equal(false); + expect(allComponentTemplateNames.includes('logs-apache.access@mappings')).to.equal(false); + }); + + it('should not remove legacy component templates if in use by index templates', async () => { + await createLegacyComponentTemplates(); + + await esClient.indices.putIndexTemplate({ + name: 'testtemplate', + index_patterns: ['nonexistentindices'], + template: {}, + composed_of: ['logs-apache.access@settings', 'logs-apache.access@mappings'], + }); + + await waitUntilLegacyComponentTemplatesCreated(); + await installUploadPackage(); + + const { component_templates: allComponentTemplates } = + await esClient.cluster.getComponentTemplate(); + const allComponentTemplateNames = allComponentTemplates.map((t) => t.name); + + expect(allComponentTemplateNames.includes('logs-apache.access@settings')).to.equal(true); + expect(allComponentTemplateNames.includes('logs-apache.access@mappings')).to.equal(true); + }); + }); +} From 953519a6c3968fd5f7f70f562732859116b6b718 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Thu, 21 Apr 2022 15:02:15 +0100 Subject: [PATCH 6/6] PR feedback --- .../elasticsearch/template/remove_legacy.ts | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts index a2f9586a7f08c..44b9756edc448 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/remove_legacy.ts @@ -25,21 +25,22 @@ export const _getLegacyComponentTemplatesForPackage = ( componentTemplates: ClusterComponentTemplate[], installablePackage: InstallablePackage ): string[] => { - const namesMap: Map = new Map(); + const legacyNamesLookup: Set = new Set(); // fill a map with all possible @mappings and @settings component // template names for fast lookup below. installablePackage.data_streams?.forEach((ds) => { LEGACY_TEMPLATE_SUFFIXES.forEach((suffix) => { - namesMap.set(getComponentTemplateWithSuffix(ds, suffix)); + legacyNamesLookup.add(getComponentTemplateWithSuffix(ds, suffix)); }); }); return componentTemplates.reduce((legacyTemplates, componentTemplate) => { - if (!namesMap.has(componentTemplate.name)) return legacyTemplates; + if (!legacyNamesLookup.has(componentTemplate.name)) return legacyTemplates; - if (componentTemplate.component_template._meta?.package?.name !== installablePackage.name) + if (componentTemplate.component_template._meta?.package?.name !== installablePackage.name) { return legacyTemplates; + } return legacyTemplates.concat(componentTemplate.name); }, []); @@ -84,15 +85,16 @@ export const _getIndexTemplatesToUsedByMap = ( return lookupMap; }; -const _getAllComponentTemplates = async (esClient: ElasticsearchClient) => - esClient.cluster.getComponentTemplate().then((result) => result.component_templates); +const _getAllComponentTemplates = async (esClient: ElasticsearchClient) => { + const { component_templates: componentTemplates } = await esClient.cluster.getComponentTemplate(); -const _getAllIndexTemplatesWithComposedOf = async (esClient: ElasticsearchClient) => - esClient.indices - .getIndexTemplate() - .then((result) => - result.index_templates.filter((tmpl) => tmpl.index_template.composed_of?.length) - ); + return componentTemplates; +}; + +const _getAllIndexTemplatesWithComposedOf = async (esClient: ElasticsearchClient) => { + const { index_templates: indexTemplates } = await esClient.indices.getIndexTemplate(); + return indexTemplates.filter((tmpl) => tmpl.index_template.composed_of?.length); +}; export const _filterComponentTemplatesInUse = ({ componentTemplateNames,