Skip to content

Commit

Permalink
Optimize package installation performance, phase 2 (elastic#131627)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshdover
joshdover authored May 6, 2022
1 parent fadd817 commit 8aa9241
Show file tree
Hide file tree
Showing 8 changed files with 196 additions and 300 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@
* 2.0.
*/

export { installPipelines, isTopLevelPipeline } from './install';
export { prepareToInstallPipelines, isTopLevelPipeline } from './install';

export { deletePreviousPipelines, deletePipeline } from './remove';
87 changes: 43 additions & 44 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/install.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@
*/

import type { TransportRequestOptions } from '@elastic/elasticsearch';
import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server';
import type { ElasticsearchClient, Logger } from '@kbn/core/server';

import { ElasticsearchAssetType } from '../../../../types';
import type { EsAssetReference, RegistryDataStream, InstallablePackage } from '../../../../types';
import { getAsset, getPathParts } from '../../archive';
import type { ArchiveEntry } from '../../archive';
import { updateEsAssetReferences } from '../../packages/install';
import {
FLEET_FINAL_PIPELINE_CONTENT,
FLEET_FINAL_PIPELINE_ID,
Expand All @@ -36,23 +35,23 @@ export const isTopLevelPipeline = (path: string) => {
);
};

export const installPipelines = async (
export const prepareToInstallPipelines = (
installablePackage: InstallablePackage,
paths: string[],
esClient: ElasticsearchClient,
savedObjectsClient: SavedObjectsClientContract,
logger: Logger,
esReferences: EsAssetReference[]
) => {
paths: string[]
): {
assetsToAdd: EsAssetReference[];
install: (esClient: ElasticsearchClient, logger: Logger) => Promise<void>;
} => {
// unlike other ES assets, pipeline names are versioned so after a template is updated
// it can be created pointing to the new template, without removing the old one and effecting data
// so do not remove the currently installed pipelines here
const dataStreams = installablePackage.data_streams;
const { name: pkgName, version: pkgVersion } = installablePackage;
const { version: pkgVersion } = installablePackage;
const pipelinePaths = paths.filter((path) => isPipeline(path));
const topLevelPipelinePaths = paths.filter((path) => isTopLevelPipeline(path));

if (!dataStreams?.length && topLevelPipelinePaths.length === 0) return [];
if (!dataStreams?.length && topLevelPipelinePaths.length === 0)
return { assetsToAdd: [], install: () => Promise.resolve() };

// get and save pipeline refs before installing pipelines
let pipelineRefs = dataStreams
Expand Down Expand Up @@ -85,41 +84,41 @@ export const installPipelines = async (

pipelineRefs = [...pipelineRefs, ...topLevelPipelineRefs];

esReferences = await updateEsAssetReferences(savedObjectsClient, pkgName, esReferences, {
return {
assetsToAdd: pipelineRefs,
});

const pipelines = dataStreams
? dataStreams.reduce<Array<Promise<EsAssetReference[]>>>((acc, dataStream) => {
if (dataStream.ingest_pipeline) {
acc.push(
installAllPipelines({
dataStream,
esClient,
logger,
paths: pipelinePaths,
installablePackage,
})
);
}
return acc;
}, [])
: [];

if (topLevelPipelinePaths) {
pipelines.push(
installAllPipelines({
dataStream: undefined,
esClient,
logger,
paths: topLevelPipelinePaths,
installablePackage,
})
);
}
install: async (esClient, logger) => {
const pipelines = dataStreams
? dataStreams.reduce<Array<Promise<EsAssetReference[]>>>((acc, dataStream) => {
if (dataStream.ingest_pipeline) {
acc.push(
installAllPipelines({
dataStream,
esClient,
logger,
paths: pipelinePaths,
installablePackage,
})
);
}
return acc;
}, [])
: [];

if (topLevelPipelinePaths) {
pipelines.push(
installAllPipelines({
dataStream: undefined,
esClient,
logger,
paths: topLevelPipelinePaths,
installablePackage,
})
);
}

await Promise.all(pipelines);
return esReferences;
await Promise.all(pipelines);
},
};
};

export function rewriteIngestPipeline(
Expand Down
140 changes: 20 additions & 120 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,19 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { loggerMock } from '@kbn/logging-mocks';

import { createAppContextStartContractMock } from '../../../../mocks';
import { appContextService } from '../../..';

import type { RegistryDataStream } from '../../../../types';
import type { Field } from '../../fields/field';

import { installTemplate } from './install';
import { prepareTemplate } from './install';

describe('EPM install', () => {
describe('EPM index template install', () => {
beforeEach(async () => {
appContextService.start(createAppContextStartContractMock());
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockImplementation(() =>
elasticsearchServiceMock.createSuccessTransportRequestPromise({ index_templates: [] })
);

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix not set', async () => {
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -43,29 +32,14 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixUnset,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixUnset });
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] });

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to false', async () => {
const dataStreamDatasetIsPrefixFalse = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -82,29 +56,15 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixFalse = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixFalse = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixFalse,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixFalse });

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixFalse);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixFalse]);
});

it('tests installPackage to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
esClient.indices.getIndexTemplate.mockResponse({ index_templates: [] });

const fields: Field[] = [];
it('tests prepareTemplate to use correct priority and index_patterns for data stream with dataset_is_prefix set to true', async () => {
const dataStreamDatasetIsPrefixTrue = {
type: 'metrics',
dataset: 'package.dataset',
Expand All @@ -121,71 +81,11 @@ describe('EPM install', () => {
};
const templateIndexPatternDatasetIsPrefixTrue = 'metrics-package.dataset.*-*';
const templatePriorityDatasetIsPrefixTrue = 150;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixTrue,
packageVersion: pkg.version,
packageName: pkg.name,
});

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;

expect(sentTemplate).toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
});

it('tests installPackage remove the aliases property if the property existed', async () => {
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;

esClient.indices.getIndexTemplate.mockResponse({
index_templates: [
{
name: 'metrics-package.dataset',
// @ts-expect-error not full interface
index_template: {
index_patterns: ['metrics-package.dataset-*'],
template: { aliases: {} },
},
},
],
});

const fields: Field[] = [];
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
} as RegistryDataStream;
const pkg = {
name: 'package',
version: '0.0.1',
};
const templateIndexPatternDatasetIsPrefixUnset = 'metrics-package.dataset-*';
const templatePriorityDatasetIsPrefixUnset = 200;
await installTemplate({
esClient,
logger: loggerMock.create(),
fields,
dataStream: dataStreamDatasetIsPrefixUnset,
packageVersion: pkg.version,
packageName: pkg.name,
});
const {
indexTemplate: { indexTemplate },
} = prepareTemplate({ pkg, dataStream: dataStreamDatasetIsPrefixTrue });

const sentTemplate = (
esClient.indices.putIndexTemplate.mock.calls[0][0] as estypes.IndicesPutIndexTemplateRequest
).body;
expect(sentTemplate).toBeDefined();
expect(sentTemplate?.template?.aliases).not.toBeDefined();
expect(sentTemplate?.priority).toBe(templatePriorityDatasetIsPrefixUnset);
expect(sentTemplate?.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixUnset]);
expect(indexTemplate.priority).toBe(templatePriorityDatasetIsPrefixTrue);
expect(indexTemplate.index_patterns).toEqual([templateIndexPatternDatasetIsPrefixTrue]);
});
});
Loading

0 comments on commit 8aa9241

Please sign in to comment.