From 9b8b7b56c1926c0b88edca31e5b5bc494a68f607 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 2 Aug 2022 20:49:15 -0400 Subject: [PATCH] [Fleet] Support dynamic_template mappings from object field (#137772) (cherry picked from commit 76c55a2d0158a259498fd86ebab21aecf0b4ceaa) --- .../plugins/fleet/common/types/models/epm.ts | 1 + .../__snapshots__/template.test.ts.snap | 100 +++++- .../epm/elasticsearch/template/install.ts | 17 +- .../epm/elasticsearch/template/mappings.ts | 48 +++ .../elasticsearch/template/template.test.ts | 11 + .../epm/elasticsearch/template/template.ts | 302 ++++++++++++------ .../fields/__snapshots__/field.test.ts.snap | 58 ++++ .../fleet/server/services/epm/fields/field.ts | 1 + .../tests/cockroachdb_dynamic_templates.yml | 37 +++ 9 files changed, 456 insertions(+), 119 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 2a97719c811c1..8c8e6288d474e 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -483,6 +483,7 @@ export type PackageAssetReference = Pick & { export interface IndexTemplateMappings { properties: any; + dynamic_templates?: any; } // This is an index template v2, see https://github.com/elastic/elasticsearch/issues/53101 diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index efd51d5e0d997..758d0e6d1bc11 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -56,6 +56,59 @@ exports[`EPM template tests loading base.yml: base.yml 1`] = ` } `; +exports[`EPM template tests loading cockroachdb_dynamic_templates.yml: cockroachdb_dynamic_templates.yml 1`] = ` +{ + "properties": {}, + "dynamic_templates": [ + { + "cockroachdb.status.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "cockroachdb.status.labels.*" + } + }, + { + "cockroachdb.status.*.value": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.value" + } + }, + { + "cockroachdb.status.*.counter": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.counter" + } + }, + { + "cockroachdb.status.*.rate": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.rate" + } + }, + { + "cockroachdb.status.*.histogram": { + "mapping": { + "type": "histogram" + }, + "match_mapping_type": "*", + "path_match": "cockroachdb.status.*.histogram" + } + } + ] +} +`; + exports[`EPM template tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` { "properties": { @@ -830,9 +883,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` "ignore_above": 2048, "type": "keyword" }, - "env": { - "type": "object" - }, "cpu": { "properties": { "user": { @@ -1026,9 +1076,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } - }, - "percpu": { - "type": "object" } } }, @@ -1341,13 +1388,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` }, "failed": { "type": "long" - }, - "states": { - "properties": { - "*": { - "type": "object" - } - } } } }, @@ -1405,9 +1445,6 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } }, - "user": { - "properties": {} - }, "summary": { "properties": { "all": { @@ -1540,6 +1577,35 @@ exports[`EPM template tests loading system.yml: system.yml 1`] = ` } } } - } + }, + "dynamic_templates": [ + { + "system.process.env": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.process.env.*" + } + }, + { + "system.process.cgroup.cpuacct.percpu": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "system.process.cgroup.cpuacct.percpu.*" + } + }, + { + "system.raid.disks.states.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.raid.disks.states.*" + } + } + ] } `; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 91f291d25d6be..c39366edab519 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { merge } from 'lodash'; +import { merge, concat, uniqBy, omit } from 'lodash'; import Boom from '@hapi/boom'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; @@ -241,6 +241,15 @@ function buildComponentTemplates(params: { const templateSettings = merge(defaultSettings, indexTemplateSettings); + const indexTemplateMappings = registryElasticsearch?.['index_template.mappings'] ?? {}; + + const mappingsProperties = merge(mappings.properties, indexTemplateMappings.properties ?? {}); + + const mappingsDynamicTemplates = uniqBy( + concat(mappings.dynamic_templates ?? [], indexTemplateMappings.dynamic_templates ?? []), + (dynampingTemplate) => Object.keys(dynampingTemplate)[0] + ); + templatesMap[packageTemplateName] = { template: { settings: { @@ -256,7 +265,11 @@ function buildComponentTemplates(params: { }, }, }, - mappings: merge(mappings, registryElasticsearch?.['index_template.mappings'] ?? {}), + mappings: { + properties: mappingsProperties, + dynamic_templates: mappingsDynamicTemplates.length ? mappingsDynamicTemplates : undefined, + ...omit(indexTemplateMappings, 'properties', 'dynamic_templates'), + }, }, _meta, }; diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts new file mode 100644 index 0000000000000..a398f4fde99d9 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/mappings.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Field } from '../../fields/field'; + +const DEFAULT_SCALING_FACTOR = 1000; + +interface Properties { + [key: string]: any; +} + +export function getDefaultProperties(field: Field): Properties { + const properties: Properties = {}; + + if (field.index !== undefined) { + properties.index = field.index; + } + if (field.doc_values !== undefined) { + properties.doc_values = field.doc_values; + } + if (field.copy_to) { + properties.copy_to = field.copy_to; + } + + return properties; +} + +export function scaledFloat(field: Field): Properties { + const fieldProps = getDefaultProperties(field); + fieldProps.type = 'scaled_float'; + fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; + if (field.metric_type) { + fieldProps.time_series_metric = field.metric_type; + } + + return fieldProps; +} + +export function histogram(field: Field): Properties { + const fieldProps = getDefaultProperties(field); + fieldProps.type = 'histogram'; + + return fieldProps; +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index d565a22347b39..5cd0081d20cdf 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -137,6 +137,17 @@ describe('EPM template', () => { expect(mappings).toMatchSnapshot(path.basename(ymlPath)); }); + it('tests loading cockroachdb_dynamic_templates.yml', () => { + const ymlPath = path.join(__dirname, '../../fields/tests/cockroachdb_dynamic_templates.yml'); + const fieldsYML = readFileSync(ymlPath, 'utf-8'); + const fields: Field[] = safeLoad(fieldsYML); + const processedFields = processFields(fields); + + const mappings = generateMappings(processedFields); + + expect(mappings).toMatchSnapshot(path.basename(ymlPath)); + }); + it('tests processing long field with index false', () => { const longWithIndexFalseYml = ` - name: longIndexFalse diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index d6f203f29c2d1..e0ea50b6420dd 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -24,6 +24,8 @@ import { import { getESAssetMetadata } from '../meta'; import { retryTransientEsErrors } from '../retry'; +import { getDefaultProperties, histogram, scaledFloat } from './mappings'; + interface Properties { [key: string]: any; } @@ -40,7 +42,7 @@ export interface CurrentDataStream { replicated: boolean; indexTemplate: IndexTemplate; } -const DEFAULT_SCALING_FACTOR = 1000; + const DEFAULT_IGNORE_ABOVE = 1024; // see discussion in https://github.com/elastic/kibana/issues/88307 @@ -103,6 +105,64 @@ export function getTemplate({ * @param fields */ export function generateMappings(fields: Field[]): IndexTemplateMappings { + const dynamicTemplates: Array> = []; + const dynamicTemplateNames = new Set(); + + const { properties } = _generateMappings(fields, { + addDynamicMapping: (dynamicMapping: { + path: string; + matchingType: string; + pathMatch: string; + properties: string; + }) => { + const name = dynamicMapping.path; + if (dynamicTemplateNames.has(name)) { + return; + } + + const dynamicTemplate: Properties = { + mapping: dynamicMapping.properties, + }; + + if (dynamicMapping.matchingType) { + dynamicTemplate.match_mapping_type = dynamicMapping.matchingType; + } + + if (dynamicMapping.pathMatch) { + dynamicTemplate.path_match = dynamicMapping.pathMatch; + } + dynamicTemplateNames.add(name); + dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate }); + }, + }); + + return dynamicTemplates.length + ? { + properties, + dynamic_templates: dynamicTemplates, + } + : { properties }; +} + +/** + * Generate mapping takes the given nested fields array and creates the Elasticsearch + * mapping properties out of it. + * + * This assumes that all fields with dotted.names have been expanded in a previous step. + * + * @param fields + */ +function _generateMappings( + fields: Field[], + ctx: { + addDynamicMapping: any; + groupFieldName?: string; + } +): { + properties: IndexTemplateMappings['properties']; + hasNonDynamicTemplateMappings: boolean; +} { + let hasNonDynamicTemplateMappings = false; const props: Properties = {}; // TODO: this can happen when the fields property in fields.yml is present but empty // Maybe validation should be moved to fields/field.ts @@ -111,101 +171,159 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings { // If type is not defined, assume keyword const type = field.type || 'keyword'; - let fieldProps = getDefaultProperties(field); + if (type === 'object' && field.object_type) { + const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name; + const pathMatch = path.includes('*') ? path : `${path}.*`; - switch (type) { - case 'group': - fieldProps = { ...generateMappings(field.fields!), ...generateDynamicAndEnabled(field) }; - break; - case 'group-nested': - fieldProps = { - ...generateMappings(field.fields!), - ...generateNestedProps(field), - type: 'nested', - }; - break; - case 'integer': - fieldProps.type = 'long'; - break; - case 'scaled_float': - fieldProps.type = 'scaled_float'; - fieldProps.scaling_factor = field.scaling_factor || DEFAULT_SCALING_FACTOR; - if (field.metric_type) { - fieldProps.time_series_metric = field.metric_type; - } - break; - case 'text': - const textMapping = generateTextMapping(field); - fieldProps = { ...fieldProps, ...textMapping, type: 'text' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'keyword': - const keywordMapping = generateKeywordMapping(field); - fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'wildcard': - const wildcardMapping = generateWildcardMapping(field); - fieldProps = { ...fieldProps, ...wildcardMapping, type: 'wildcard' }; - if (field.multi_fields) { - fieldProps.fields = generateMultiFields(field.multi_fields); - } - break; - case 'constant_keyword': - fieldProps.type = field.type; - if (field.value) { - fieldProps.value = field.value; - } - break; - case 'object': - fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; - break; - case 'nested': - fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; - break; - case 'array': - // this assumes array fields were validated in an earlier step - // adding an array field with no object_type would result in an error - // when the template is added to ES - if (field.object_type) { - fieldProps.type = field.object_type; - } - break; - case 'alias': - // this assumes alias fields were validated in an earlier step - // adding a path to a field that doesn't exist would result in an error - // when the template is added to ES. - fieldProps.type = 'alias'; - fieldProps.path = field.path; - break; - default: - fieldProps.type = type; - } + let dynProperties: Properties = getDefaultProperties(field); + let matchingType: string | undefined; + switch (field.object_type) { + case 'histogram': + dynProperties = histogram(field); + matchingType = field.object_type_mapping_type ?? '*'; + break; + case 'text': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'string'; + break; + case 'keyword': + dynProperties.type = field.object_type; + matchingType = field.object_type_mapping_type ?? 'string'; + break; + case 'byte': + case 'double': + case 'float': + case 'long': + case 'short': + case 'boolean': + dynProperties = { + type: field.object_type, + }; + matchingType = field.object_type_mapping_type ?? field.object_type; + default: + break; + } + + if (dynProperties && matchingType) { + ctx.addDynamicMapping({ + path, + pathMatch, + matchingType, + properties: dynProperties, + }); + } + } else { + let fieldProps = getDefaultProperties(field); - const fieldHasMetaProps = META_PROP_KEYS.some((key) => key in field); - if (fieldHasMetaProps) { switch (type) { case 'group': + const mappings = _generateMappings(field.fields!, { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }); + if (!mappings.hasNonDynamicTemplateMappings) { + return; + } + + fieldProps = { + properties: mappings.properties, + ...generateDynamicAndEnabled(field), + }; + break; case 'group-nested': + fieldProps = { + properties: _generateMappings(field.fields!, { + ...ctx, + groupFieldName: ctx.groupFieldName + ? `${ctx.groupFieldName}.${field.name}` + : field.name, + }).properties, + ...generateNestedProps(field), + type: 'nested', + }; + break; + case 'integer': + fieldProps.type = 'long'; + break; + case 'scaled_float': + fieldProps = scaledFloat(field); + break; + case 'text': + const textMapping = generateTextMapping(field); + fieldProps = { ...fieldProps, ...textMapping, type: 'text' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } break; - default: { - const meta = {}; - if ('metric_type' in field) Reflect.set(meta, 'metric_type', field.metric_type); - if ('unit' in field) Reflect.set(meta, 'unit', field.unit); - fieldProps.meta = meta; + case 'object': + fieldProps = { ...fieldProps, ...generateDynamicAndEnabled(field), type: 'object' }; + break; + case 'keyword': + const keywordMapping = generateKeywordMapping(field); + fieldProps = { ...fieldProps, ...keywordMapping, type: 'keyword' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } + break; + case 'wildcard': + const wildcardMapping = generateWildcardMapping(field); + fieldProps = { ...fieldProps, ...wildcardMapping, type: 'wildcard' }; + if (field.multi_fields) { + fieldProps.fields = generateMultiFields(field.multi_fields); + } + break; + case 'constant_keyword': + fieldProps.type = field.type; + if (field.value) { + fieldProps.value = field.value; + } + break; + case 'nested': + fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; + break; + case 'array': + // this assumes array fields were validated in an earlier step + // adding an array field with no object_type would result in an error + // when the template is added to ES + if (field.object_type) { + fieldProps.type = field.object_type; + } + break; + case 'alias': + // this assumes alias fields were validated in an earlier step + // adding a path to a field that doesn't exist would result in an error + // when the template is added to ES. + fieldProps.type = 'alias'; + fieldProps.path = field.path; + break; + default: + fieldProps.type = type; + } + + const fieldHasMetaProps = META_PROP_KEYS.some((key) => key in field); + if (fieldHasMetaProps) { + switch (type) { + case 'group': + case 'group-nested': + break; + default: { + const meta = {}; + if ('metric_type' in field) Reflect.set(meta, 'metric_type', field.metric_type); + if ('unit' in field) Reflect.set(meta, 'unit', field.unit); + fieldProps.meta = meta; + } } } - } - props[field.name] = fieldProps; + props[field.name] = fieldProps; + hasNonDynamicTemplateMappings = true; + } }); } - return { properties: props }; + return { properties: props, hasNonDynamicTemplateMappings }; } function generateDynamicAndEnabled(field: Field) { @@ -295,22 +413,6 @@ function generateWildcardMapping(field: Field): IndexTemplateMapping { return mapping; } -function getDefaultProperties(field: Field): Properties { - const properties: Properties = {}; - - if (field.index !== undefined) { - properties.index = field.index; - } - if (field.doc_values !== undefined) { - properties.doc_values = field.doc_values; - } - if (field.copy_to) { - properties.copy_to = field.copy_to; - } - - return properties; -} - /** * Generates the template name out of the given information */ diff --git a/x-pack/plugins/fleet/server/services/epm/fields/__snapshots__/field.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/fields/__snapshots__/field.test.ts.snap index 78f1fbc528696..a1c8a2a801ca8 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/__snapshots__/field.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/epm/fields/__snapshots__/field.test.ts.snap @@ -64,6 +64,64 @@ exports[`tests loading fields.yml: base.yml 1`] = ` ] `; +exports[`tests loading fields.yml: cockroachdb_dynamic_templates.yml 1`] = ` +[ + { + "name": "cockroachdb", + "type": "group", + "fields": [ + { + "name": "status", + "type": "group", + "release": "beta", + "fields": [ + { + "name": "labels", + "type": "object", + "object_type": "keyword", + "description": "Prometheus metric labels\\n" + }, + { + "name": "*", + "type": "group", + "fields": [ + { + "name": "value", + "type": "object", + "object_type": "double", + "object_type_mapping_type": "*", + "description": "Prometheus gauge metric\\n" + }, + { + "name": "counter", + "type": "object", + "object_type": "double", + "object_type_mapping_type": "*", + "description": "Prometheus counter metric\\n" + }, + { + "name": "rate", + "type": "object", + "object_type": "double", + "object_type_mapping_type": "*", + "description": "Prometheus rated counter metric\\n" + }, + { + "name": "histogram", + "type": "object", + "object_type": "histogram", + "object_type_mapping_type": "*", + "description": "Prometheus histogram metric" + } + ] + } + ] + } + ] + } +] +`; + exports[`tests loading fields.yml: coredns.logs.yml 1`] = ` [ { diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.ts index 0e00840b0c74e..8d784e0cffc22 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.ts @@ -30,6 +30,7 @@ export interface Field { search_analyzer?: string; ignore_above?: number; object_type?: string; + object_type_mapping_type?: string; scaling_factor?: number; dynamic?: 'strict' | boolean; include_in_parent?: boolean; diff --git a/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml b/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml new file mode 100644 index 0000000000000..d9ab38e07d7d6 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/fields/tests/cockroachdb_dynamic_templates.yml @@ -0,0 +1,37 @@ +- name: cockroachdb.status + type: group + release: beta + fields: + - name: labels + type: object + object_type: keyword + description: > + Prometheus metric labels + +- name: cockroachdb.status.*.value + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus gauge metric + +- name: cockroachdb.status.*.counter + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus counter metric + +- name: cockroachdb.status.*.rate + type: object + object_type: double + object_type_mapping_type: '*' + description: > + Prometheus rated counter metric + +- name: cockroachdb.status.*.histogram + type: object + object_type: histogram + object_type_mapping_type: '*' + description: >- + Prometheus histogram metric