From 275fb9731df36c0dd1954963f3af420c7f601564 Mon Sep 17 00:00:00 2001 From: Sandra Gonzales Date: Wed, 1 Jul 2020 11:53:24 -0400 Subject: [PATCH] [Ingest Manager] add _meta field to index templates (#70319) * add _meta field to index templates * fix typescript issue * make package an object Co-authored-by: Elastic Machine --- .../ingest_manager/common/types/models/epm.ts | 1 + .../__snapshots__/template.test.ts.snap | 18 +++++++++++ .../epm/elasticsearch/template/install.ts | 11 ++++++- .../elasticsearch/template/template.test.ts | 28 ++++++++++++++--- .../epm/elasticsearch/template/template.ts | 30 ++++++++++++++----- .../apis/template.ts | 7 ++++- 6 files changed, 81 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index f45924c30faad..b001b994187e1 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -276,6 +276,7 @@ export interface IndexTemplate { data_stream: { timestamp_field: string; }; + _meta: object; } export interface TemplateRef { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index b1212cf3a6535..635742c82f9a4 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -93,6 +93,12 @@ exports[`tests loading base.yml: base.yml 1`] = ` }, "data_stream": { "timestamp_field": "@timestamp" + }, + "_meta": { + "package": { + "name": "nginx" + }, + "managed_by": "ingest-manager" } } `; @@ -190,6 +196,12 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` }, "data_stream": { "timestamp_field": "@timestamp" + }, + "_meta": { + "package": { + "name": "coredns" + }, + "managed_by": "ingest-manager" } } `; @@ -1671,6 +1683,12 @@ exports[`tests loading system.yml: system.yml 1`] = ` }, "data_stream": { "timestamp_field": "@timestamp" + }, + "_meta": { + "package": { + "name": "system" + }, + "managed_by": "ingest-manager" } } `; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts index 9d0b6b5d078ad..a318aecf347d6 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts @@ -153,6 +153,7 @@ export async function installTemplateForDataset({ fields, dataset, packageVersion: pkg.version, + packageName: pkg.name, }); } @@ -161,11 +162,13 @@ export async function installTemplate({ fields, dataset, packageVersion, + packageName, }: { callCluster: CallESAsCurrentUser; fields: Field[]; dataset: Dataset; packageVersion: string; + packageName: string; }): Promise { const mappings = generateMappings(processFields(fields)); const templateName = generateTemplateName(dataset); @@ -177,7 +180,13 @@ export async function installTemplate({ packageVersion, }); } - const template = getTemplate(dataset.type, templateName, mappings, pipelineName); + const template = getTemplate({ + type: dataset.type, + templateName, + mappings, + pipelineName, + packageName, + }); // TODO: Check return values for errors const callClusterParams: { method: string; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index cacf84381dd88..73a6767f6b947 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -24,7 +24,12 @@ expect.addSnapshotSerializer({ test('get template', () => { const templateName = 'logs-nginx-access-abcd'; - const template = getTemplate('logs', templateName, { properties: {} }); + const template = getTemplate({ + type: 'logs', + templateName, + packageName: 'nginx', + mappings: { properties: {} }, + }); expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); }); @@ -35,7 +40,12 @@ test('tests loading base.yml', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate('logs', 'foo', mappings); + const template = getTemplate({ + type: 'logs', + templateName: 'foo', + packageName: 'nginx', + mappings, + }); expect(template).toMatchSnapshot(path.basename(ymlPath)); }); @@ -47,7 +57,12 @@ test('tests loading coredns.logs.yml', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate('logs', 'foo', mappings); + const template = getTemplate({ + type: 'logs', + templateName: 'foo', + packageName: 'coredns', + mappings, + }); expect(template).toMatchSnapshot(path.basename(ymlPath)); }); @@ -59,7 +74,12 @@ test('tests loading system.yml', () => { const processedFields = processFields(fields); const mappings = generateMappings(processedFields); - const template = getTemplate('metrics', 'whatsthis', mappings); + const template = getTemplate({ + type: 'metrics', + templateName: 'whatsthis', + packageName: 'system', + mappings, + }); expect(template).toMatchSnapshot(path.basename(ymlPath)); }); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 9e8f327d520e3..06c07da6cd77a 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -37,13 +37,20 @@ const DEFAULT_IGNORE_ABOVE = 1024; * * @param indexPattern String with the index pattern */ -export function getTemplate( - type: string, - templateName: string, - mappings: IndexTemplateMappings, - pipelineName?: string | undefined -): IndexTemplate { - const template = getBaseTemplate(type, templateName, mappings); +export function getTemplate({ + type, + templateName, + mappings, + pipelineName, + packageName, +}: { + type: string; + templateName: string; + mappings: IndexTemplateMappings; + pipelineName?: string | undefined; + packageName: string; +}): IndexTemplate { + const template = getBaseTemplate(type, templateName, mappings, packageName); if (pipelineName) { template.template.settings.index.default_pipeline = pipelineName; } @@ -236,7 +243,8 @@ export function generateESIndexPatterns(datasets: Dataset[] | undefined): Record function getBaseTemplate( type: string, templateName: string, - mappings: IndexTemplateMappings + mappings: IndexTemplateMappings, + packageName: string ): IndexTemplate { return { // This takes precedence over all index templates installed with the 'base' package @@ -297,6 +305,12 @@ function getBaseTemplate( data_stream: { timestamp_field: '@timestamp', }, + _meta: { + package: { + name: packageName, + }, + managed_by: 'ingest-manager', + }, }; } diff --git a/x-pack/test/ingest_manager_api_integration/apis/template.ts b/x-pack/test/ingest_manager_api_integration/apis/template.ts index 53fffef4cc0e2..8911dd28dc243 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/template.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/template.ts @@ -22,7 +22,12 @@ export default function ({ getService }: FtrProviderContext) { // This test was inspired by https://github.com/elastic/kibana/blob/master/x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js describe('template', async () => { it('can be loaded', async () => { - const template = getTemplate('logs', indexPattern, mappings); + const template = getTemplate({ + type: 'logs', + templateName, + mappings, + packageName: 'system', + }); // This test is not an API integration test with Kibana // We want to test here if the template is valid and for this we need a running ES instance.