6.x index template data type fallback (#1171)

CHANGELOG.next.md

@@ -48,6 +48,7 @@ Thanks, you're awesome :-) -->
 
 * Added a notice highlighting that the `tracing` fields are not nested under the
   namespace `tracing.` #1162
+* ES 6.x template data types will fallback to supported types. #1171
 
 #### Deprecated
 

scripts/generators/es_template.py

@@ -4,6 +4,8 @@
 
 from os.path import join
 from generators import ecs_helpers
+from schema.cleaner import field_or_multi_field_datatype_defaults
+from schema.oss import TYPE_FALLBACKS
 
 
 def generate(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file):
@@ -93,7 +95,9 @@ def generate_template_version(elasticsearch_version, mappings_section, out_dir,
     else:
         template = default_template_settings()
     if elasticsearch_version == 6:
-        template['mappings'] = {'_doc': mappings_section}
+        es6_mappings_section = copy.deepcopy(mappings_section)
+        es6_type_fallback(es6_mappings_section['properties'])
+        template['mappings'] = {'_doc': es6_mappings_section}
     else:
         template['mappings'] = mappings_section
 
@@ -144,3 +148,22 @@ def default_mapping_settings(ecs_version):
         ],
         "properties": {}
     }
+
+
+def es6_type_fallback(mappings):
+    '''
+    Visits each leaf in mappings object and fallback to an
+    Elasticsearch 6.x supported type.
+
+    Since a field like `wildcard` won't have the same defaults as
+    a `keyword` field, we must add any missing defaults.
+    '''
+
+    for (name, details) in mappings.items():
+        if 'type' in details:
+            fallback_type = TYPE_FALLBACKS.get(details['type'])
+            if fallback_type:
+                mappings[name]['type'] = fallback_type
+                field_or_multi_field_datatype_defaults(mappings[name])
+        if 'properties' in details:
+            es6_type_fallback(details['properties'])

scripts/tests/test_es_template.py

@@ -157,6 +157,71 @@ def test_constant_keyword_no_value(self):
         exp = {'type': 'constant_keyword'}
         self.assertEqual(es_template.entry_for(test_map), exp)
 
+    def test_es6_fallback_base_case_wildcard(self):
+        test_map = {
+            "field": {
+                "name": "field",
+                "type": "wildcard"
+            }
+        }
+
+        exp = {
+            "field": {
+                "name": "field",
+                "type": "keyword",
+                "ignore_above": 1024
+            }
+        }
+
+        es_template.es6_type_fallback(test_map)
+        self.assertEqual(test_map, exp)
+
+    def test_es6_fallback_recursive_case_wildcard(self):
+        test_map = {
+            "top_field": {
+                "properties": {
+                    "field": {
+                        "name": "field",
+                        "type": "wildcard"
+                    }
+                }
+            }
+        }
+
+        exp = {
+            "top_field": {
+                "properties": {
+                    "field": {
+                        "name": "field",
+                        "type": "keyword",
+                        "ignore_above": 1024
+                    }
+                }
+            }
+        }
+
+        es_template.es6_type_fallback(test_map)
+        self.assertEqual(test_map, exp)
+
+    def test_es6_fallback_base_case_constant_keyword(self):
+        test_map = {
+            "field": {
+                "name": "field",
+                "type": "constant_keyword"
+            }
+        }
+
+        exp = {
+            "field": {
+                "name": "field",
+                "type": "keyword",
+                "ignore_above": 1024
+            }
+        }
+
+        es_template.es6_type_fallback(test_map)
+        self.assertEqual(test_map, exp)
+
 
 if __name__ == '__main__':
     unittest.main()