diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index 9f8ac783ad..df39066ed8 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -47,6 +47,7 @@ Thanks, you're awesome :-) --> * Added a notice highlighting that the `tracing` fields are not nested under the namespace `tracing.` #1162 +* ES 6.x template data types will fallback to supported types. #1171 #### Deprecated diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 086d5246b9..bb56356a5a 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -4,6 +4,8 @@ from os.path import join from generators import ecs_helpers +from schema.cleaner import field_or_multi_field_datatype_defaults +from schema.oss import TYPE_FALLBACKS def generate(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file): @@ -93,7 +95,9 @@ def generate_template_version(elasticsearch_version, mappings_section, out_dir, else: template = default_template_settings() if elasticsearch_version == 6: - template['mappings'] = {'_doc': mappings_section} + es6_mappings_section = copy.deepcopy(mappings_section) + es6_type_fallback(es6_mappings_section['properties']) + template['mappings'] = {'_doc': es6_mappings_section} else: template['mappings'] = mappings_section @@ -144,3 +148,22 @@ def default_mapping_settings(ecs_version): ], "properties": {} } + + +def es6_type_fallback(mappings): + ''' + Visits each leaf in mappings object and fallback to an + Elasticsearch 6.x supported type. + + Since a field like `wildcard` won't have the same defaults as + a `keyword` field, we must add any missing defaults. + ''' + + for (name, details) in mappings.items(): + if 'type' in details: + fallback_type = TYPE_FALLBACKS.get(details['type']) + if fallback_type: + mappings[name]['type'] = fallback_type + field_or_multi_field_datatype_defaults(mappings[name]) + if 'properties' in details: + es6_type_fallback(details['properties']) diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index 43ee4d276f..50142e4ff6 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -157,6 +157,71 @@ def test_constant_keyword_no_value(self): exp = {'type': 'constant_keyword'} self.assertEqual(es_template.entry_for(test_map), exp) + def test_es6_fallback_base_case_wildcard(self): + test_map = { + "field": { + "name": "field", + "type": "wildcard" + } + } + + exp = { + "field": { + "name": "field", + "type": "keyword", + "ignore_above": 1024 + } + } + + es_template.es6_type_fallback(test_map) + self.assertEqual(test_map, exp) + + def test_es6_fallback_recursive_case_wildcard(self): + test_map = { + "top_field": { + "properties": { + "field": { + "name": "field", + "type": "wildcard" + } + } + } + } + + exp = { + "top_field": { + "properties": { + "field": { + "name": "field", + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + + es_template.es6_type_fallback(test_map) + self.assertEqual(test_map, exp) + + def test_es6_fallback_base_case_constant_keyword(self): + test_map = { + "field": { + "name": "field", + "type": "constant_keyword" + } + } + + exp = { + "field": { + "name": "field", + "type": "keyword", + "ignore_above": 1024 + } + } + + es_template.es6_type_fallback(test_map) + self.assertEqual(test_map, exp) + if __name__ == '__main__': unittest.main()