Merge mappings for composable index templates (#58709)

This PR implements recursive mapping merging for composable index templates.

When creating an index, we perform the following:
* Add each component template mapping in order, merging each one in after the
last.
* Merge in the index template mappings (if present).
* Merge in the mappings on the index request itself (if present).

Some principles:
* All 'structural' changes are disallowed (but everything else is fine). An
object mapper can never be changed between `type: object` and `type: nested`. A
field mapper can never be changed to an object mapper, and vice versa.
* Generally, each section is merged recursively. This includes `object`
mappings, as well as root options like `dynamic_templates` and `meta`. Once we
reach 'leaf components' like field definitions, they always overwrite an
existing one instead of being merged.

Relates to #53101.

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml

@@ -200,3 +200,89 @@
         index: eggplant
 
   - match: {eggplant.settings.index.number_of_shards: "3"}
+
+---
+"Index template mapping merging":
+  - skip:
+      version: " - 7.8.99"
+      reason: "index template v2 mapping merging not available before 7.9"
+      features: allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: red
+        body:
+          template:
+            mappings:
+              properties:
+                object1.red:
+                  type: keyword
+                object2.red:
+                  type: keyword
+
+  - do:
+      cluster.put_component_template:
+        name: blue
+        body:
+          template:
+            mappings:
+              properties:
+                object2.red:
+                  type: text
+                object1.blue:
+                  type: text
+                object2.blue:
+                  type: text
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template] has index patterns [baz*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: blue
+        body:
+          index_patterns: ["purple-index"]
+          composed_of: ["red", "blue"]
+          template:
+            mappings:
+              properties:
+                object2.blue:
+                  type: integer
+                object1.purple:
+                  type: integer
+                object2.purple:
+                  type: integer
+                nested:
+                  type: nested
+                  include_in_root: true
+
+  - do:
+      indices.create:
+        index: purple-index
+        body:
+          mappings:
+            properties:
+              object2.purple:
+                type: double
+              object3.purple:
+                type: double
+              nested:
+                type: nested
+                include_in_root: false
+                include_in_parent: true
+
+  - do:
+      indices.get:
+        index: purple-index
+
+  - match: {purple-index.mappings.properties.object1.properties.red: {type: keyword}}
+  - match: {purple-index.mappings.properties.object1.properties.blue: {type: text}}
+  - match: {purple-index.mappings.properties.object1.properties.purple: {type: integer}}
+
+  - match: {purple-index.mappings.properties.object2.properties.red: {type: text}}
+  - match: {purple-index.mappings.properties.object2.properties.blue: {type: integer}}
+  - match: {purple-index.mappings.properties.object2.properties.purple: {type: double}}
+
+  - match: {purple-index.mappings.properties.object3.properties.purple: {type: double}}
+
+  - is_false: purple-index.mappings.properties.nested.include_in_root
+  - is_true: purple-index.mappings.properties.nested.include_in_parent

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -28,22 +28,21 @@
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.AliasMetadata;
 import org.elasticsearch.cluster.metadata.AliasValidator;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -58,7 +57,6 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.resolveV2Mappings;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
@@ -172,13 +170,6 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
                                            final AliasValidator aliasValidator) throws Exception {
         Settings settings = resolveSettings(simulatedState.metadata(), matchingTemplate);
 
-        // empty request mapping as the user can't specify any explicit mappings via the simulate api
-        Map<String, Map<String, Object>> mappings = resolveV2Mappings("{}", simulatedState, matchingTemplate, xContentRegistry);
-        assert mappings.size() == 1 : "expected always to have 1 mapping type but there were " + mappings.size();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> docMappings = (Map<String, Object>) mappings.get(MapperService.SINGLE_MAPPING_NAME);
-        String mappingsJson = Strings.toString(XContentFactory.jsonBuilder().map(docMappings));
-
         List<Map<String, AliasMetadata>> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulatedState.metadata(),
             matchingTemplate);
 
@@ -203,8 +194,28 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri
                 // the context is only used for validation so it's fine to pass fake values for the
                 // shard id and the current timestamp
                 tempIndexService.newQueryShardContext(0, null, () -> 0L, null)));
+        Map<String, AliasMetadata> aliasesByName = aliases.stream().collect(
+            Collectors.toMap(AliasMetadata::getAlias, Function.identity()));
 
-        return new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson),
-            aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())));
+        // empty request mapping as the user can't specify any explicit mappings via the simulate api
+        List<Map<String, Map<String, Object>>> mappings = MetadataCreateIndexService.collectV2Mappings(
+            Collections.emptyMap(), simulatedState, matchingTemplate, xContentRegistry);
+
+        CompressedXContent mergedMapping = indicesService.<CompressedXContent, Exception>withTempIndexService(indexMetadata,
+            tempIndexService -> {
+                MapperService mapperService = tempIndexService.mapperService();
+                for (Map<String, Map<String, Object>> mapping : mappings) {
+                    if (!mapping.isEmpty()) {
+                        assert mapping.size() == 1 : mapping;
+                        Map.Entry<String, Map<String, Object>> entry = mapping.entrySet().iterator().next();
+                        mapperService.merge(entry.getKey(), entry.getValue(), MapperService.MergeReason.INDEX_TEMPLATE);
+                    }
+                }
+
+                DocumentMapper documentMapper = mapperService.documentMapper();
+                return documentMapper != null ? documentMapper.mappingSource() : null;
+            });
+
+        return new Template(settings, mergedMapping, aliasesByName);
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -27,12 +27,13 @@
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.MapperService;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -244,28 +245,24 @@ public TimestampField(StreamInput in) throws IOException {
         }
 
         /**
-         * Force fully inserts the timestamp field mapping into the provided mapping.
-         * Existing mapping definitions for the timestamp field will be completely overwritten.
-         * Takes into account if the name of the timestamp field is nested.
-         *
-         * @param mappings The mapping to update
+         * Creates a map representing the full timestamp field mapping, taking into
+         * account if the timestamp field is nested under object mappers (its path
+         * contains dots).
          */
-        public void insertTimestampFieldMapping(Map<String, Object> mappings) {
-            assert mappings.containsKey("_doc");
-
+        public Map<String, Map<String, Object>> getTimestampFieldMapping() {
             String mappingPath = convertFieldPathToMappingPath(name);
-            String parentObjectFieldPath = "_doc." + mappingPath.substring(0, mappingPath.lastIndexOf('.'));
+            String parentObjectFieldPath = mappingPath.substring(0, mappingPath.lastIndexOf('.'));
             String leafFieldName = mappingPath.substring(mappingPath.lastIndexOf('.') + 1);
 
-            Map<String, Object> changes = new HashMap<>();
-            Map<String, Object> current = changes;
+            Map<String, Object> result = new HashMap<>();
+            Map<String, Object> current = result;
             for (String key : parentObjectFieldPath.split("\\.")) {
                 Map<String, Object> map = new HashMap<>();
                 current.put(key, map);
                 current = map;
             }
             current.put(leafFieldName, fieldMapping);
-            XContentHelper.update(mappings, changes, false);
+            return Collections.singletonMap(MapperService.SINGLE_MAPPING_NAME, result);
         }
 
         @Override