diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index aa28857e144ee..269749c8d8477 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -173,6 +173,8 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) { private static Map findFieldMappingsByType(Predicate fieldPredicate, DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) { + //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather + // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup Map fieldMappings = new HashMap<>(); final MappingLookup mappingLookup = documentMapper.mappers(); for (String field : request.fields()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 7dbde31f13949..2d0bab6aab2f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -279,15 +279,7 @@ public final FieldMapper merge(Mapper mergeWith) { Conflicts conflicts = new Conflicts(name()); builder.merge((FieldMapper) mergeWith, conflicts); conflicts.check(); - return builder.build(parentPath(name())); - } - - private static ContentPath parentPath(String name) { - int endPos = name.lastIndexOf("."); - if (endPos == -1) { - return new ContentPath(0); - } - return new ContentPath(name.substring(0, endPos)); + return builder.build(Builder.parentPath(name())); } protected void checkIncomingMergeType(FieldMapper mergeWith) { @@ -483,7 +475,7 @@ public List copyToFields() { /** * Serializes a parameter */ - protected interface Serializer { + public interface Serializer { void serialize(XContentBuilder builder, String name, T value) throws IOException; } @@ -936,7 +928,7 @@ protected String buildFullName(ContentPath contentPath) { /** * Writes the current builder parameter values as XContent */ - protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { for (Parameter parameter : getParameters()) { parameter.toXContent(builder, includeDefaults); } @@ -1010,6 +1002,14 @@ public final void parse(String name, ParserContext parserContext, Map fullNameToFieldType = new HashMap<>(); /** @@ -47,7 +46,8 @@ final class FieldTypeLookup { private final DynamicKeyFieldTypeLookup dynamicKeyLookup; FieldTypeLookup(Collection fieldMappers, - Collection fieldAliasMappers) { + Collection fieldAliasMappers, + Collection runtimeFieldTypes) { Map dynamicKeyMappers = new HashMap<>(); for (FieldMapper fieldMapper : fieldMappers) { @@ -77,6 +77,11 @@ final class FieldTypeLookup { fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path)); } + for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) { + //this will override concrete fields with runtime fields that have the same name + fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType); + } + this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index d7cfe22950ff9..2bae141fc9d12 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -59,6 +59,7 @@ class ParserContext { private final Function similarityLookupService; private final Function typeParsers; + private final Function runtimeTypeParsers; private final Version indexVersionCreated; private final Supplier queryShardContextSupplier; private final DateFormatter dateFormatter; @@ -69,6 +70,7 @@ class ParserContext { public ParserContext(Function similarityLookupService, Function typeParsers, + Function runtimeTypeParsers, Version indexVersionCreated, Supplier queryShardContextSupplier, DateFormatter dateFormatter, @@ -78,6 +80,7 @@ public ParserContext(Function similarityLookupServic BooleanSupplier idFieldDataEnabled) { this.similarityLookupService = similarityLookupService; this.typeParsers = typeParsers; + this.runtimeTypeParsers = runtimeTypeParsers; this.indexVersionCreated = indexVersionCreated; this.queryShardContextSupplier = queryShardContextSupplier; this.dateFormatter = dateFormatter; @@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() { protected Function typeParsers() { return typeParsers; } + protected Function runtimeTypeParsers() { return runtimeTypeParsers; } + protected Function similarityLookupService() { return similarityLookupService; } /** @@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) { static class MultiFieldParserContext extends ParserContext { MultiFieldParserContext(ParserContext in) { - super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier, - in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled); + super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated, + in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, + in.idFieldDataEnabled); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 8aab2f4d57de8..eba38b31c00e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -149,8 +149,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.mapperRegistry = mapperRegistry; Function parserContextFunction = dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get, - indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings, - idFieldDataEnabled); + mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter, + scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled); this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction); Map metadataMapperParsers = mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 481c5497c88e1..7226b06a28661 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -32,7 +32,6 @@ import java.util.stream.Stream; public final class MappingLookup { - /** Full field name to mapper */ private final Map fieldMappers; private final Map objectMappers; @@ -50,24 +49,24 @@ public static MappingLookup fromMapping(Mapping mapping) { newFieldMappers.add(metadataMapper); } } - collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers); - return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length); + for (Mapper child : mapping.root) { + collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers); + } + return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, + mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length); } private static void collect(Mapper mapper, Collection objectMappers, Collection fieldMappers, Collection fieldAliasMappers) { - if (mapper instanceof RootObjectMapper) { - // root mapper isn't really an object mapper - } else if (mapper instanceof ObjectMapper) { + if (mapper instanceof ObjectMapper) { objectMappers.add((ObjectMapper)mapper); } else if (mapper instanceof FieldMapper) { fieldMappers.add((FieldMapper)mapper); } else if (mapper instanceof FieldAliasMapper) { fieldAliasMappers.add((FieldAliasMapper) mapper); } else { - throw new IllegalStateException("Unrecognized mapper type [" + - mapper.getClass().getSimpleName() + "]."); + throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "]."); } for (Mapper child : mapper) { @@ -78,6 +77,7 @@ private static void collect(Mapper mapper, Collection objectMapper public MappingLookup(Collection mappers, Collection objectMappers, Collection aliasMappers, + Collection runtimeFieldTypes, int metadataFieldCount) { Map fieldMappers = new HashMap<>(); Map indexAnalyzers = new HashMap<>(); @@ -114,7 +114,7 @@ public MappingLookup(Collection mappers, } } - this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers); + this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes); this.fieldMappers = Collections.unmodifiableMap(fieldMappers); this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 5181352945604..0b5cf05c881d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; @@ -34,11 +35,14 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter; @@ -62,6 +66,7 @@ public static class Builder extends ObjectMapper.Builder { protected Explicit dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false); protected Explicit dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false); protected Explicit numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false); + protected final Map runtimeFieldTypes = new HashMap<>(); public Builder(String name, Version indexCreatedVersion) { super(name, indexCreatedVersion); @@ -83,6 +88,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) { return this; } + public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) { + this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType); + return this; + } + @Override public RootObjectMapper build(ContentPath contentPath) { return (RootObjectMapper) super.build(contentPath); @@ -92,7 +102,7 @@ public RootObjectMapper build(ContentPath contentPath) { protected ObjectMapper createMapper(String name, String fullPath, Explicit enabled, Nested nested, Dynamic dynamic, Map mappers, Version indexCreatedVersion) { assert !nested.isNested(); - return new RootObjectMapper(name, enabled, dynamic, mappers, + return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes, dynamicDateTimeFormatters, dynamicTemplates, dateDetection, numericDetection, indexCreatedVersion); @@ -127,7 +137,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare } } - public static class TypeParser extends ObjectMapper.TypeParser { + static final class TypeParser extends ObjectMapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { @@ -145,8 +155,8 @@ public Mapper.Builder parse(String name, Map node, ParserContext return builder; } - protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, - ParserContext parserContext) { + @SuppressWarnings("unchecked") + private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) { if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) { if (fieldNode instanceof List) { List formatters = new ArrayList<>(); @@ -200,6 +210,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam } else if (fieldName.equals("numeric_detection")) { builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true); return true; + } else if (fieldName.equals("runtime")) { + if (fieldNode instanceof Map) { + RuntimeFieldType.parseRuntimeFields((Map) fieldNode, parserContext, builder::addRuntime); + return true; + } else { + throw new ElasticsearchParseException("runtime must be a map type"); + } } return false; } @@ -209,11 +226,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam private Explicit dateDetection; private Explicit numericDetection; private Explicit dynamicTemplates; + private final Map runtimeFieldTypes; RootObjectMapper(String name, Explicit enabled, Dynamic dynamic, Map mappers, + Map runtimeFieldTypes, Explicit dynamicDateTimeFormatters, Explicit dynamicTemplates, Explicit dateDetection, Explicit numericDetection, Version indexCreatedVersion) { super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion); + this.runtimeFieldTypes = runtimeFieldTypes; this.dynamicTemplates = dynamicTemplates; this.dynamicDateTimeFormatters = dynamicDateTimeFormatters; this.dateDetection = dateDetection; @@ -233,23 +253,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) { return update; } - public boolean dateDetection() { + boolean dateDetection() { return this.dateDetection.value(); } - public boolean numericDetection() { + boolean numericDetection() { return this.numericDetection.value(); } - public DateFormatter[] dynamicDateTimeFormatters() { + DateFormatter[] dynamicDateTimeFormatters() { return dynamicDateTimeFormatters.value(); } - public DynamicTemplate[] dynamicTemplates() { + DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } - @SuppressWarnings("rawtypes") + Collection runtimeFieldTypes() { + return runtimeFieldTypes.values(); + } + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { return findTemplateBuilder(context, name, matchType, null); } @@ -265,7 +288,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format * @return a mapper builder, or null if there is no template for such a field */ - @SuppressWarnings("rawtypes") private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); if (dynamicTemplate == null) { @@ -328,6 +350,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) { this.dynamicTemplates = mergeWithObject.dynamicTemplates; } } + + this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes); } @Override @@ -358,6 +382,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr if (numericDetection.explicit() || includeDefaults) { builder.field("numeric_detection", numericDetection.value()); } + + if (runtimeFieldTypes.size() > 0) { + builder.startObject("runtime"); + List sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted( + Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList()); + for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) { + fieldType.toXContent(builder, params); + } + builder.endObject(); + } } private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java new file mode 100644 index 0000000000000..75fd74532a15b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/RuntimeFieldType.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.function.Consumer; + +/** + * Base implementation for a runtime field that can be defined as part of the runtime section of the index mappings + */ +public abstract class RuntimeFieldType extends MappedFieldType implements ToXContentFragment { + + protected RuntimeFieldType(String name, Map meta) { + super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(name()); + builder.field("type", typeName()); + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); + doXContentBody(builder, includeDefaults); + builder.endObject(); + return builder; + } + + /** + * Prints out the parameters that subclasses expose + */ + protected abstract void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException; + + /** + * Parser for a runtime field. Creates the appropriate {@link RuntimeFieldType} for a runtime field, + * as defined in the runtime section of the index mappings. + */ + public interface Parser { + RuntimeFieldType parse(String name, Map node, Mapper.TypeParser.ParserContext parserContext) + throws MapperParsingException; + } + + public static void parseRuntimeFields(Map node, + Mapper.TypeParser.ParserContext parserContext, + Consumer runtimeFieldTypeConsumer) { + Iterator> iterator = node.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String fieldName = entry.getKey(); + if (entry.getValue() instanceof Map) { + @SuppressWarnings("unchecked") + Map propNode = new HashMap<>(((Map) entry.getValue())); + Object typeNode = propNode.get("type"); + String type; + if (typeNode == null) { + throw new MapperParsingException("No type specified for runtime field [" + fieldName + "]"); + } else { + type = typeNode.toString(); + } + Parser typeParser = parserContext.runtimeTypeParsers().apply(type); + if (typeParser == null) { + throw new MapperParsingException("No handler for type [" + type + + "] declared on runtime field [" + fieldName + "]"); + } + runtimeFieldTypeConsumer.accept(typeParser.parse(fieldName, propNode, parserContext)); + propNode.remove("type"); + DocumentMapperParser.checkNoRemainingFields(fieldName, propNode, parserContext.indexVersionCreated()); + iterator.remove(); + } else { + throw new MapperParsingException("Expected map for runtime field [" + fieldName + "] definition but got a " + + fieldName.getClass().getName()); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index ee380cc7868a2..243be3f8ca0a3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -26,7 +26,6 @@ import org.apache.lucene.search.join.BitSetProducer; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.Client; @@ -57,6 +56,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.query.support.NestedScope; @@ -195,7 +195,7 @@ public QueryShardContext( ), allowExpensiveQueries, valuesSourceRegistry, - parseRuntimeMappings(runtimeMappings, mapperService, indexSettings) + parseRuntimeMappings(runtimeMappings, mapperService) ); } @@ -396,21 +396,12 @@ public MappedFieldType buildAnonymousFieldType(String type) { "[unmapped_type:string] should be replaced with [unmapped_type:keyword]"); type = "keyword"; } - return buildFieldType(type, "__anonymous_" + type, Collections.emptyMap(), mapperService.parserContext(), indexSettings); - } - - private static MappedFieldType buildFieldType( - String type, - String field, - Map node, - Mapper.TypeParser.ParserContext parserContext, - IndexSettings indexSettings - ) { + Mapper.TypeParser.ParserContext parserContext = mapperService.parserContext(); Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new IllegalArgumentException("No mapper found for type [" + type + "]"); } - Mapper.Builder builder = typeParser.parse(field, node, parserContext); + Mapper.Builder builder = typeParser.parse("__anonymous_", Collections.emptyMap(), parserContext); Mapper mapper = builder.build(new ContentPath(0)); if (mapper instanceof FieldMapper) { return ((FieldMapper)mapper).fieldType(); @@ -674,25 +665,14 @@ public BigArrays bigArrays() { } private static Map parseRuntimeMappings( - Map mappings, - MapperService mapperService, - IndexSettings indexSettings + Map runtimeMappings, + MapperService mapperService ) { - Map runtimeMappings = new HashMap<>(); - for (Map.Entry entry : mappings.entrySet()) { - String field = entry.getKey(); - if (entry.getValue() instanceof Map == false) { - throw new ElasticsearchParseException("runtime mappings must be a map type"); - } - @SuppressWarnings("unchecked") - Map node = new HashMap<>((Map) entry.getValue()); - // Replace the type until we have native support for the runtime section - Object oldRuntimeType = node.put("runtime_type", node.remove("type")); - if (oldRuntimeType != null) { - throw new ElasticsearchParseException("use [type] in [runtime_mappings] instead of [runtime_type]"); - } - runtimeMappings.put(field, buildFieldType("runtime", field, node, mapperService.parserContext(), indexSettings)); + Map runtimeFieldTypes = new HashMap<>(); + if (runtimeMappings.isEmpty() == false) { + RuntimeFieldType.parseRuntimeFields(new HashMap<>(runtimeMappings), mapperService.parserContext(), + runtimeFieldType -> runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType)); } - return runtimeMappings; + return Collections.unmodifiableMap(runtimeFieldTypes); } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 57a8f1383329c..53ba191f3ffff 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -83,8 +84,8 @@ public class IndicesModule extends AbstractModule { private final MapperRegistry mapperRegistry; public IndicesModule(List mapperPlugins) { - this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getMetadataMappers(mapperPlugins), - getFieldFilter(mapperPlugins)); + this.mapperRegistry = new MapperRegistry(getMappers(mapperPlugins), getRuntimeFieldTypes(mapperPlugins), + getMetadataMappers(mapperPlugins), getFieldFilter(mapperPlugins)); registerBuiltinWritables(); } @@ -144,9 +145,21 @@ public static Map getMappers(List mappe return Collections.unmodifiableMap(mappers); } + private static Map getRuntimeFieldTypes(List mapperPlugins) { + Map runtimeParsers = new LinkedHashMap<>(); + for (MapperPlugin mapperPlugin : mapperPlugins) { + for (Map.Entry entry : mapperPlugin.getRuntimeFieldTypes().entrySet()) { + if (runtimeParsers.put(entry.getKey(), entry.getValue()) != null) { + throw new IllegalArgumentException("Runtime field type [" + entry.getKey() + "] is already registered"); + } + } + } + return Collections.unmodifiableMap(runtimeParsers); + } + private static final Map builtInMetadataMappers = initBuiltInMetadataMappers(); - private static Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); + private static final Set builtInMetadataFields = Collections.unmodifiableSet(builtInMetadataMappers.keySet()); private static Map initBuiltInMetadataMappers() { Map builtInMetadataMappers; diff --git a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java index 5fa385da449ad..0c43b7139eeac 100644 --- a/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java +++ b/server/src/main/java/org/elasticsearch/indices/mapper/MapperRegistry.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import java.util.Collections; @@ -37,14 +38,16 @@ public final class MapperRegistry { private final Map mapperParsers; + private final Map runtimeFieldTypeParsers; private final Map metadataMapperParsers; private final Map metadataMapperParsers6x; private final Function> fieldFilter; - public MapperRegistry(Map mapperParsers, + public MapperRegistry(Map mapperParsers, Map runtimeFieldTypeParsers, Map metadataMapperParsers, Function> fieldFilter) { this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); + this.runtimeFieldTypeParsers = runtimeFieldTypeParsers; this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); // add the _all field mapper for indices created in 6x Map metadata6x = new LinkedHashMap<>(); @@ -62,6 +65,10 @@ public Map getMapperParsers() { return mapperParsers; } + public Map getRuntimeFieldTypeParsers() { + return runtimeFieldTypeParsers; + } + /** * Return a map of the meta mappers that have been registered. The * returned map uses the name of the field as a key. diff --git a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java index 5edf994b32ea4..8544c75a575ee 100644 --- a/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/MapperPlugin.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import java.util.Collections; import java.util.Map; @@ -43,6 +44,10 @@ default Map getMappers() { return Collections.emptyMap(); } + default Map getRuntimeFieldTypes() { + return Collections.emptyMap(); + } + /** * Returns additional metadata mapper implementations added by this plugin. *

diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 5f66552afdf19..9c5097b06743c 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1030,7 +1030,7 @@ public SearchSourceBuilder pointInTimeBuilder(PointInTimeBuilder builder) { * Mappings specified on this search request that override built in mappings. */ public Map runtimeMappings() { - return runtimeMappings; + return Collections.unmodifiableMap(runtimeMappings); } /** diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 8a75d0d28dec2..e6da2484cf870 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -562,7 +562,7 @@ protected String contentType() { } }; MappingLookup mappingLookup = new MappingLookup(Arrays.asList(mockedTimestampField, dateFieldMapper), - Collections.emptyList(), Collections.emptyList(), 0); + Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), 0); ClusterService clusterService = ClusterServiceUtils.createClusterService(testThreadPool); Environment env = mock(Environment.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java index 48aac8a713d33..725d2ac71a1a9 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexUpgradeServiceTests.java @@ -153,7 +153,7 @@ private MetadataIndexUpgradeService getMetadataIndexUpgradeService() { return new MetadataIndexUpgradeService( Settings.EMPTY, xContentRegistry(), - new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), + new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new SystemIndices(Collections.singletonMap("system-plugin", Collections.singletonList(new SystemIndexDescriptor(".system", "a system index")))), diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index ffc08f9e1d689..553349428f90a 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -19,15 +19,30 @@ package org.elasticsearch.index; +import org.apache.lucene.search.Query; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; +import java.util.function.Supplier; + import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class IndexSortSettingsTests extends ESTestCase { private static IndexSettings indexSettings(Settings settings) { @@ -129,4 +144,50 @@ public void testInvalidMissing() { assertThat(exc.getMessage(), containsString("Illegal missing value:[default]," + " must be one of [_last, _first]")); } + + public void testIndexSorting() { + IndexSettings indexSettings = indexSettings(Settings.builder().put("index.sort.field", "field").build()); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null); + NoneCircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService(); + final IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, cache, circuitBreakerService, null); + MappedFieldType fieldType = new RuntimeFieldType("field", Collections.emptyMap()) { + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public String typeName() { + return null; + } + + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + searchLookup.get(); + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new UnsupportedOperationException(); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + throw new UnsupportedOperationException(); + } + }; + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> config.buildIndexSort( + field -> fieldType, + (ft, searchLookupSupplier) -> indexFieldDataService.getForField(ft, "index", searchLookupSupplier) + ) + ); + assertEquals("docvalues not found for index sort field:[field]", iae.getMessage()); + assertThat(iae.getCause(), instanceOf(UnsupportedOperationException.class)); + assertEquals("index sorting not supported on runtime field [field]", iae.getCause().getMessage()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index aa86888cdefe9..f01bcc169b191 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -92,7 +92,8 @@ private CodecService createCodecService() throws IOException { IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); SimilarityService similarityService = new SimilarityService(settings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; - MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), + MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, () -> null, () -> false, null); return new CodecService(service, LogManager.getLogger("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 829550a5439e2..2b386472f8816 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -119,6 +119,7 @@ public void testAnalyzers() throws IOException { Arrays.asList(fieldMapper1, fieldMapper2), Collections.emptyList(), Collections.emptyList(), + Collections.emptyList(), 0); assertAnalyzes(mappingLookup.indexAnalyzer(), "field1", "index1"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index 975a22872dfb6..16d2d97bb5cee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -39,7 +39,9 @@ public void testDuplicateFieldAliasAndObject() { new MappingLookup( Collections.emptyList(), singletonList(objectMapper), - singletonList(aliasMapper), 0)); + singletonList(aliasMapper), + emptyList(), + 0)); assertEquals("Alias [some.path] is defined both as an object and an alias", e.getMessage()); } @@ -52,7 +54,9 @@ public void testDuplicateFieldAliasAndConcreteField() { new MappingLookup( Arrays.asList(field, invalidField), emptyList(), - singletonList(invalidAlias), 0)); + singletonList(invalidAlias), + emptyList(), + 0)); assertEquals("Alias [invalid] is defined both as an alias and a concrete field", e.getMessage()); } @@ -65,7 +69,9 @@ public void testAliasThatRefersToAlias() { MappingLookup mappers = new MappingLookup( singletonList(field), emptyList(), - Arrays.asList(alias, invalidAlias), 0); + Arrays.asList(alias, invalidAlias), + emptyList(), + 0); alias.validate(mappers); MapperParsingException e = expectThrows(MapperParsingException.class, () -> invalidAlias.validate(mappers)); @@ -81,7 +87,9 @@ public void testAliasThatRefersToItself() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -96,7 +104,9 @@ public void testAliasWithNonExistentPath() { MappingLookup mappers = new MappingLookup( emptyList(), emptyList(), - singletonList(invalidAlias), 0); + singletonList(invalidAlias), + emptyList(), + 0); invalidAlias.validate(mappers); }); @@ -112,6 +122,7 @@ public void testFieldAliasWithNestedScope() { singletonList(createFieldMapper("nested", "field")), singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -124,6 +135,7 @@ public void testFieldAliasWithDifferentObjectScopes() { singletonList(createFieldMapper("object1", "field")), Arrays.asList(createObjectMapper("object1"), createObjectMapper("object2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); } @@ -137,6 +149,7 @@ public void testFieldAliasWithNestedTarget() { singletonList(createFieldMapper("nested", "field")), Collections.singletonList(objectMapper), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); @@ -155,6 +168,7 @@ public void testFieldAliasWithDifferentNestedScopes() { singletonList(createFieldMapper("nested1", "field")), Arrays.asList(createNestedObjectMapper("nested1"), createNestedObjectMapper("nested2")), singletonList(aliasMapper), + emptyList(), 0); aliasMapper.validate(mappers); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 5a8543324ea75..d0277665315f1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -34,7 +34,7 @@ public class FieldTypeLookupTests extends ESTestCase { public void testEmpty() { - FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); assertNull(lookup.get("foo")); Collection names = lookup.simpleMatchToFullName("foo"); assertNotNull(names); @@ -45,23 +45,37 @@ public void testEmpty() { public void testFilter() { Collection fieldMappers = List.of(new MockFieldMapper("field"), new MockFieldMapper("test")); Collection fieldAliases = singletonList(new FieldAliasMapper("alias", "alias", "test")); - FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(fieldMappers, fieldAliases); - Iterable allFieldTypes = fieldTypeLookup.filter(ft -> true); - assertEquals(2, size(allFieldTypes)); - for (MappedFieldType allFieldType : allFieldTypes) { - assertThat(allFieldType, instanceOf(MockFieldMapper.FakeFieldType.class)); + Collection runtimeFields = List.of(new TestRuntimeField("runtime"), new TestRuntimeField("field")); + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(fieldMappers, fieldAliases, runtimeFields); + assertEquals(3, size(fieldTypeLookup.filter(ft -> true))); + for (MappedFieldType fieldType : fieldTypeLookup.filter(ft -> true)) { + if (fieldType.name().equals("test")) { + assertThat(fieldType, instanceOf(MockFieldMapper.FakeFieldType.class)); + } + if (fieldType.name().equals("field") || fieldType.name().equals("runtime")) { + assertThat(fieldType, instanceOf(TestRuntimeField.class)); + } } assertEquals(0, size(fieldTypeLookup.filter(ft -> false))); - Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("field")); - assertEquals(1, size(fieldIterable)); - MappedFieldType field = fieldIterable.iterator().next(); - assertEquals("field", field.name()); - assertThat(field, instanceOf(MockFieldMapper.FakeFieldType.class)); + { + Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("field")); + assertEquals(1, size(fieldIterable)); + MappedFieldType field = fieldIterable.iterator().next(); + assertEquals("field", field.name()); + assertThat(field, instanceOf(TestRuntimeField.class)); + } + { + Iterable fieldIterable = fieldTypeLookup.filter(ft -> ft.name().equals("test")); + assertEquals(1, size(fieldIterable)); + MappedFieldType field = fieldIterable.iterator().next(); + assertEquals("test", field.name()); + assertThat(field, instanceOf(MockFieldMapper.FakeFieldType.class)); + } } public void testAddNewField() { MockFieldMapper f = new MockFieldMapper("foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(f), emptyList(), Collections.emptyList()); assertNull(lookup.get("bar")); assertEquals(f.fieldType(), lookup.get("foo")); assertEquals(1, size(lookup.filter(ft -> true))); @@ -71,7 +85,8 @@ public void testAddFieldAlias() { MockFieldMapper field = new MockFieldMapper("foo"); FieldAliasMapper alias = new FieldAliasMapper("alias", "alias", "foo"); - FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(Collections.singletonList(field), Collections.singletonList(alias), + Collections.emptyList()); MappedFieldType aliasType = lookup.get("alias"); assertEquals(field.fieldType(), aliasType); @@ -84,7 +99,7 @@ public void testSimpleMatchToFullName() { FieldAliasMapper alias1 = new FieldAliasMapper("food", "food", "foo"); FieldAliasMapper alias2 = new FieldAliasMapper("barometer", "barometer", "bar"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2)); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field1, field2), Arrays.asList(alias1, alias2), Collections.emptyList()); Collection names = lookup.simpleMatchToFullName("b*"); @@ -101,7 +116,7 @@ public void testSourcePathWithMultiFields() { .addMultiField(new MockFieldMapper.Builder("field.subfield2")) .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(field), emptyList(), emptyList()); assertEquals(Set.of("field"), lookup.sourcePaths("field")); assertEquals(Set.of("field"), lookup.sourcePaths("field.subfield1")); @@ -117,13 +132,86 @@ public void testSourcePathsWithCopyTo() { .copyTo("field") .build(new ContentPath()); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(field, otherField), emptyList(), emptyList()); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field")); assertEquals(Set.of("other_field", "field"), lookup.sourcePaths("field.subfield1")); } - private int size(Iterable iterable) { + public void testRuntimeFieldsLookup() { + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(concrete), emptyList(), List.of(runtime)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(2, size(fieldTypeLookup.filter(ft -> true))); + } + + public void testRuntimeFieldOverrides() { + MockFieldMapper field = new MockFieldMapper("field"); + MockFieldMapper subfield = new MockFieldMapper("object.subfield"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField fieldOverride = new TestRuntimeField("field"); + TestRuntimeField subfieldOverride = new TestRuntimeField("object.subfield"); + TestRuntimeField runtime = new TestRuntimeField("runtime"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field, concrete, subfield), emptyList(), + List.of(fieldOverride, runtime, subfieldOverride)); + assertThat(fieldTypeLookup.get("field"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertThat(fieldTypeLookup.get("concrete"), instanceOf(MockFieldMapper.FakeFieldType.class)); + assertThat(fieldTypeLookup.get("runtime"), instanceOf(TestRuntimeField.class)); + assertEquals(4, size(fieldTypeLookup.filter(ft -> true))); + } + + public void testRuntimeFieldsSimpleMatchToFullName() { + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + java.util.Set matches = fieldTypeLookup.simpleMatchToFullName("fie*"); + assertEquals(2, matches.size()); + assertTrue(matches.contains("field1")); + assertTrue(matches.contains("field2")); + } + { + java.util.Set matches = fieldTypeLookup.simpleMatchToFullName("object.sub*"); + assertEquals(1, matches.size()); + assertTrue(matches.contains("object.subfield")); + } + } + + public void testRuntimeFieldsSourcePaths() { + //we test that runtime fields are treated like any other field by sourcePaths, although sourcePaths + // should never be called for runtime fields as they are not in _source + MockFieldMapper field1 = new MockFieldMapper("field1"); + MockFieldMapper concrete = new MockFieldMapper("concrete"); + TestRuntimeField field2 = new TestRuntimeField("field2"); + TestRuntimeField subfield = new TestRuntimeField("object.subfield"); + + FieldTypeLookup fieldTypeLookup = new FieldTypeLookup(List.of(field1, concrete), emptyList(), List.of(field2, subfield)); + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("field1"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field1")); + } + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("field2"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("field2")); + } + { + java.util.Set sourcePaths = fieldTypeLookup.sourcePaths("object.subfield"); + assertEquals(1, sourcePaths.size()); + assertTrue(sourcePaths.contains("object.subfield")); + } + } + + private static int size(Iterable iterable) { int count = 0; for (MappedFieldType fieldType : iterable) { count++; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java new file mode 100644 index 0000000000000..f8307a7fdd361 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/MappingLookupTests.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.test.ESTestCase; + +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.instanceOf; + +public class MappingLookupTests extends ESTestCase { + + public void testOnlyRuntimeField() { + MappingLookup mappingLookup = new MappingLookup(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + Collections.singletonList(new TestRuntimeField("test")), 0); + assertEquals(0, size(mappingLookup.fieldMappers())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertNull(mappingLookup.getMapper("test")); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + } + + public void testRuntimeFieldLeafOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("test"); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.emptyList(), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("test")), 0); + assertThat(mappingLookup.getMapper("test"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.fieldMappers())); + assertEquals(0, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("test"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().filter(ft -> true))); + } + + public void testSubfieldOverride() { + MockFieldMapper fieldMapper = new MockFieldMapper("object.subfield"); + ObjectMapper objectMapper = new ObjectMapper("object", "object", new Explicit<>(true, true), ObjectMapper.Nested.NO, + ObjectMapper.Dynamic.TRUE, Collections.singletonMap("object.subfield", fieldMapper), Version.CURRENT); + MappingLookup mappingLookup = new MappingLookup(Collections.singletonList(fieldMapper), Collections.singletonList(objectMapper), + Collections.emptyList(), Collections.singletonList(new TestRuntimeField("object.subfield")), 0); + assertThat(mappingLookup.getMapper("object.subfield"), instanceOf(MockFieldMapper.class)); + assertEquals(1, size(mappingLookup.fieldMappers())); + assertEquals(1, mappingLookup.objectMappers().size()); + assertThat(mappingLookup.fieldTypes().get("object.subfield"), instanceOf(TestRuntimeField.class)); + assertEquals(1, size(mappingLookup.fieldTypes().filter(ft -> true))); + } + + private static int size(Iterable iterable) { + int count = 0; + for (Object obj : iterable) { + count++; + } + return count; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 8b9efa3b08e9b..ae3ed0e69dca7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -214,7 +214,7 @@ private static TestMapper fromMapping(String mapping, Version version) { return BinaryFieldMapper.PARSER; } return null; - }, version, () -> null, null, null, + }, name -> null, version, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index 9e80227773c33..bac139782cbe0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -19,123 +19,127 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; import static org.elasticsearch.test.VersionUtils.randomVersionBetween; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; -public class RootObjectMapperTests extends ESSingleNodeTestCase { +public class RootObjectMapperTests extends MapperServiceTestCase { public void testNumericDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("numeric_detection", false) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // update with a different explicit value String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("numeric_detection", true) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); // update with an implicit value: no change String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); } public void testDateDetection() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("date_detection", true) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // update with a different explicit value String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .field("date_detection", false) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); // update with an implicit value: no change String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping2, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping2, mapperService.documentMapper().mappingSource().toString()); } public void testDateFormatters() throws Exception { MergeReason reason = randomFrom(MergeReason.MAPPING_UPDATE, MergeReason.INDEX_TEMPLATE); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_date_formats", Arrays.asList("yyyy-MM-dd")) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_date_formats", Collections.singletonList("yyyy-MM-dd")) .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // no update if formatters are not set explicitly String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), reason); - assertEquals(mapping, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping2); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_date_formats", Arrays.asList()) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_date_formats", Collections.emptyList()) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), reason); - assertEquals(mapping3, mapper.mappingSource().toString()); + merge(mapperService, reason, mapping3); + assertEquals(mapping3, mapperService.documentMapper().mappingSource().toString()); } public void testDynamicTemplates() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("dynamic_templates") .startObject() .startObject("my_template") @@ -148,27 +152,26 @@ public void testDynamicTemplates() throws Exception { .endArray() .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - assertEquals(mapping, mapper.mappingSource().toString()); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); // no update if templates are not set explicitly String mapping2 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); - assertEquals(mapping, mapper.mappingSource().toString()); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping2); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); String mapping3 = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") - .field("dynamic_templates", Arrays.asList()) + .startObject(MapperService.SINGLE_MAPPING_NAME) + .field("dynamic_templates", Collections.emptyList()) .endObject() .endObject()); - mapper = mapperService.merge("type", new CompressedXContent(mapping3), MergeReason.MAPPING_UPDATE); - assertEquals(mapping3, mapper.mappingSource().toString()); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping3); + assertEquals(mapping3, mapperService.documentMapper().mappingSource().toString()); } public void testDynamicTemplatesForIndexTemplate() throws IOException { @@ -192,7 +195,7 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { .endObject() .endArray() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createMapperService(Version.CURRENT, Settings.EMPTY, () -> true); mapperService.merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(mapping), MergeReason.INDEX_TEMPLATE); // There should be no update if templates are not set. @@ -249,7 +252,7 @@ public void testDynamicTemplatesForIndexTemplate() throws IOException { public void testIllegalFormatField() throws Exception { String dynamicMapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("dynamic_date_formats") .startArray().value("test_format").endArray() .endArray() @@ -257,41 +260,39 @@ public void testIllegalFormatField() throws Exception { .endObject()); String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startArray("date_formats") .startArray().value("test_format").endArray() .endArray() .endObject() .endObject()); - - MapperService mapperService = createIndex("test").mapperService(); for (String m : Arrays.asList(mapping, dynamicMapping)) { - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> mapperService.parse("type", new CompressedXContent(m), false)); - assertEquals("Invalid format: [[test_format]]: expected string value", e.getMessage()); + MapperParsingException e = expectThrows(MapperParsingException.class, + () -> createMapperService(MapperService.SINGLE_MAPPING_NAME, m)); + assertEquals("Failed to parse mapping [_doc]: Invalid format: [[test_format]]: expected string value", e.getMessage()); } } public void testIllegalDynamicTemplates() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder() .startObject() - .startObject("type") + .startObject(MapperService.SINGLE_MAPPING_NAME) .startObject("dynamic_templates") .endObject() .endObject() .endObject()); - MapperService mapperService = createIndex("test").mapperService(); MapperParsingException e = expectThrows(MapperParsingException.class, - () -> mapperService.parse("type", new CompressedXContent(mapping), false)); - assertEquals("Dynamic template syntax error. An array of named objects is expected.", e.getMessage()); + () -> createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping)); + assertEquals("Failed to parse mapping [_doc]: Dynamic template syntax error. An array of named objects is expected.", + e.getMessage()); } public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -307,9 +308,8 @@ public void testIllegalDynamicTemplateUnknownFieldType() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\"")); + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"type\":\"string\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{\"type\":" + "\"string\"}}], attempted to validate it with the following match_mapping_type: [[string]], " + "caused by [No mapper found for type [string]]"); @@ -319,7 +319,7 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -336,9 +336,9 @@ public void testIllegalDynamicTemplateUnknownAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\"")); + + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"foo\":\"bar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"keyword\"}}], " + "attempted to validate it with the following match_mapping_type: [[string]], " + @@ -349,7 +349,7 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -366,9 +366,9 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { mapping.endObject(); } mapping.endObject(); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"analyzer\":\"foobar\"")); + + MapperService mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"analyzer\":\"foobar\"")); assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"string\",\"mapping\":{" + "\"analyzer\":\"foobar\",\"type\":\"text\"}}], attempted to validate it with the following match_mapping_type: [[string]], " + "caused by [analyzer [foobar] has not been configured in mappings]"); @@ -376,12 +376,11 @@ public void testIllegalDynamicTemplateInvalidAttribute() throws Exception { public void testIllegalDynamicTemplateNoMappingType() throws Exception { MapperService mapperService; - { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -402,17 +401,15 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { mapping.endObject(); } mapping.endObject(); - mapperService = createIndex("test").mapperService(); - DocumentMapper mapper = - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"index_phrases\":true")); + mapperService = createMapperService(mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"index_phrases\":true")); } { boolean useMatchMappingType = randomBoolean(); XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -434,9 +431,8 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { } mapping.endObject(); - DocumentMapper mapper = - mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"foo\":\"bar\"")); + merge(mapperService, mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"foo\":\"bar\"")); if (useMatchMappingType) { assertWarnings("dynamic template [my_template] has invalid content [{\"match_mapping_type\":\"*\",\"mapping\":{" + "\"foo\":\"bar\",\"type\":\"{dynamic_type}\"}}], " + @@ -453,16 +449,11 @@ public void testIllegalDynamicTemplateNoMappingType() throws Exception { } } - @Override - protected boolean forbidPrivateIndexSettings() { - return false; - } - public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder(); mapping.startObject(); { - mapping.startObject("type"); + mapping.startObject(MapperService.SINGLE_MAPPING_NAME); mapping.startArray("dynamic_templates"); { mapping.startObject(); @@ -478,12 +469,167 @@ public void testIllegalDynamicTemplatePre7Dot7Index() throws Exception { mapping.endObject(); } mapping.endObject(); + Version createdVersion = randomVersionBetween(random(), Version.V_7_0_0, Version.V_7_6_0); - Settings indexSettings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion) - .build(); - MapperService mapperService = createIndex("test", indexSettings).mapperService(); - DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(Strings.toString(mapping)), MergeReason.MAPPING_UPDATE); - assertThat(mapper.mappingSource().toString(), containsString("\"type\":\"string\"")); + MapperService mapperService = createMapperService(createdVersion, mapping); + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString("\"type\":\"string\"")); + } + + @Override + protected Collection getPlugins() { + return Collections.singletonList(new RuntimeFieldPlugin()); + } + + public void testRuntimeSection() throws IOException { + String mapping = Strings.toString(runtimeMapping(builder -> { + builder.startObject("field1").field("type", "test").field("prop1", "value1").endObject(); + builder.startObject("field2").field("type", "test").field("prop2", "value2").endObject(); + builder.startObject("field3").field("type", "test").endObject(); + })); + MapperService mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); + } + + public void testRuntimeSectionMerge() throws IOException { + MapperService mapperService; + { + String mapping = Strings.toString(fieldMapping(b -> b.field("type", "keyword"))); + mapperService = createMapperService(MapperService.SINGLE_MAPPING_NAME, mapping); + assertEquals(mapping, mapperService.documentMapper().mappingSource().toString()); + MappedFieldType field = mapperService.fieldType("field"); + assertThat(field, instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + { + String mapping = Strings.toString(runtimeMapping(builder -> { + builder.startObject("field").field("type", "test").field("prop1", "first version").endObject(); + builder.startObject("field2").field("type", "test").endObject(); + })); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + //field overrides now the concrete field already defined + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertEquals("first version", field.prop1); + assertNull(field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + } + { + String mapping = Strings.toString(runtimeMapping( + //the existing runtime field gets updated + builder -> builder.startObject("field").field("type", "test").field("prop2", "second version").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertNull(field.prop1); + assertEquals("second version", field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + } + { + String mapping = Strings.toString(mapping(builder -> builder.startObject("concrete").field("type", "keyword").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + RuntimeField field = (RuntimeField)mapperService.fieldType("field"); + assertNull(field.prop1); + assertEquals("second version", field.prop2); + RuntimeField field2 = (RuntimeField)mapperService.fieldType("field2"); + assertNull(field2.prop1); + assertNull(field2.prop2); + MappedFieldType concrete = mapperService.fieldType("concrete"); + assertThat(concrete, instanceOf(KeywordFieldMapper.KeywordFieldType.class)); + } + { + String mapping = Strings.toString(runtimeMapping( + builder -> builder.startObject("field3").field("type", "test").field("prop1", "value").endObject())); + merge(MapperService.SINGLE_MAPPING_NAME, mapperService, mapping); + assertEquals("{\"_doc\":" + + "{\"runtime\":{" + + "\"field\":{\"type\":\"test\",\"prop2\":\"second version\"}," + + "\"field2\":{\"type\":\"test\"}," + + "\"field3\":{\"type\":\"test\",\"prop1\":\"value\"}}," + + "\"properties\":{" + + "\"concrete\":{\"type\":\"keyword\"}," + + "\"field\":{\"type\":\"keyword\"}}}}", + mapperService.documentMapper().mappingSource().toString()); + } + } + + public void testRuntimeSectionNonRuntimeType() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "keyword")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No handler for type [keyword] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionHandlerNotFound() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "unknown")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No handler for type [unknown] declared on runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionMissingType() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> {}); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: No type specified for runtime field [field]", e.getMessage()); + } + + public void testRuntimeSectionWrongFormat() throws IOException { + XContentBuilder mapping = runtimeMapping(builder -> builder.field("field", "value")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: Expected map for runtime field [field] definition but got a java.lang.String", + e.getMessage()); + } + + public void testRuntimeSectionRemainingField() throws IOException { + XContentBuilder mapping = runtimeFieldMapping(builder -> builder.field("type", "test").field("unsupported", "value")); + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createMapperService(mapping)); + assertEquals("Failed to parse mapping [_doc]: Mapping definition for [field] has unsupported parameters: " + + "[unsupported : value]", e.getMessage()); + } + + private static class RuntimeFieldPlugin extends Plugin implements MapperPlugin { + @Override + public Map getRuntimeFieldTypes() { + return Collections.singletonMap("test", (name, node, parserContext) -> { + Object prop1 = node.remove("prop1"); + Object prop2 = node.remove("prop2"); + return new RuntimeField(name, prop1 == null ? null : prop1.toString(), prop2 == null ? null : prop2.toString()); + }); + } + } + + private static final class RuntimeField extends RuntimeFieldType { + private final String prop1; + private final String prop2; + + protected RuntimeField(String name, String prop1, String prop2) { + super(name, Collections.emptyMap()); + this.prop1 = prop1; + this.prop2 = prop2; + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return "test"; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + if (prop1 != null) { + builder.field("prop1", prop1); + } + if (prop2 != null) { + builder.field("prop2", prop2); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java new file mode 100644 index 0000000000000..15a03e4de6dc7 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TestRuntimeField.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.util.Collections; + +public class TestRuntimeField extends RuntimeFieldType { + public TestRuntimeField(String name) { + super(name, Collections.emptyMap()); + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults) { + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + return null; + } + + @Override + public String typeName() { + return null; + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return null; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java index 83383f6343ded..00a058c9c0255 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TypeParsersTests.java @@ -79,8 +79,8 @@ public void testMultiFieldWithinMultiField() throws IOException { IndexAnalyzers indexAnalyzers = new IndexAnalyzers(defaultAnalyzers(), Collections.emptyMap(), Collections.emptyMap()); MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, Version.CURRENT, null, - null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { + Mapper.TypeParser.ParserContext olderContext = new Mapper.TypeParser.ParserContext(null, type -> typeParser, type -> null, + Version.CURRENT, null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), () -> { throw new UnsupportedOperationException(); }); diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index eaed67f424abb..1cf382cdd7c80 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.Version; @@ -38,6 +39,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -49,34 +51,40 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData; -import org.elasticsearch.index.mapper.ContentPath; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.IndexFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.Mapper.TypeParser; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MockFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.RuntimeFieldType; +import org.elasticsearch.index.mapper.TestRuntimeField; import org.elasticsearch.index.mapper.TextFieldMapper; -import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.LeafSearchLookup; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.BiFunction; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; @@ -84,9 +92,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class QueryShardContextTests extends ESTestCase { @@ -216,40 +221,40 @@ public void testIndexSortedOnField() { } public void testFielddataLookupSelfReference() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field that depends on itself e.g. field: doc['field'] return leafLookup.doc().get(field).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: field -> field", iae.getMessage()); } public void testFielddataLookupLooseLoop() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['1'] if (field.equals("4")) { return leafLookup.doc().get("1").toString(); } return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 1", iae.getMessage()); } public void testFielddataLookupTerminatesInLoop() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { //simulate a runtime field cycle: 1: doc['2'] 2: doc['3'] 3: doc['4'] 4: doc['4'] if (field.equals("4")) { return leafLookup.doc().get("4").toString(); } return leafLookup.doc().get(Integer.toString(Integer.parseInt(field) + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Cyclic dependency detected while resolving runtime fields: 1 -> 2 -> 3 -> 4 -> 4", iae.getMessage()); } public void testFielddataLookupSometimesLoop() throws IOException { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { if (docId == 0) { return field + "_" + docId; } else { @@ -260,7 +265,7 @@ public void testFielddataLookupSometimesLoop() throws IOException { int i = Integer.parseInt(field.substring(field.length() - 1)); return leafLookup.doc().get("field" + (i + 1)).toString(); } - }); + })); List values = collect("field1", queryShardContext, new TermQuery(new Term("indexed_field", "first"))); assertEquals(Collections.singletonList("field1_0"), values); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("field1", queryShardContext)); @@ -269,16 +274,16 @@ public void testFielddataLookupSometimesLoop() throws IOException { } public void testFielddataLookupBeyondMaxDepth() { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { int i = Integer.parseInt(field); return leafLookup.doc().get(Integer.toString(i + 1)).toString(); - }); + })); IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> collect("1", queryShardContext)); assertEquals("Field requires resolving too many dependent fields: 1 -> 2 -> 3 -> 4 -> 5 -> 6", iae.getMessage()); } public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { int i = Integer.parseInt(field.substring(field.length() - 1)); if (i == 5) { return "test"; @@ -286,13 +291,13 @@ public void testFielddataLookupReferencesBelowMaxDepth() throws IOException { ScriptDocValues scriptDocValues = leafLookup.doc().get("field" + (i + 1)); return scriptDocValues.get(0).toString() + docId; } - }); + })); assertEquals(Arrays.asList("test0000", "test1111"), collect("field1", queryShardContext)); } public void testFielddataLookupOneFieldManyReferences() throws IOException { int numFields = randomIntBetween(5, 20); - QueryShardContext queryShardContext = createQueryShardContext("uuid", null, (field, leafLookup, docId) -> { + QueryShardContext queryShardContext = createQueryShardContext("uuid", null, fieldTypeLookup((field, leafLookup, docId) -> { if (field.equals("field")) { StringBuilder value = new StringBuilder(); for (int i = 0; i < numFields; i++) { @@ -302,7 +307,7 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { } else { return "test" + docId; } - }); + })); StringBuilder expectedFirstDoc = new StringBuilder(); StringBuilder expectedSecondDoc = new StringBuilder(); for (int i = 0; i < numFields; i++) { @@ -312,86 +317,80 @@ public void testFielddataLookupOneFieldManyReferences() throws IOException { assertEquals(Arrays.asList(expectedFirstDoc.toString(), expectedSecondDoc.toString()), collect("field", queryShardContext)); } - public void testRuntimeFields() throws IOException { - MapperService mapperService = mockMapperService("test", org.elasticsearch.common.collect.List.of(new MapperPlugin() { - @Override - public Map getMappers() { - return org.elasticsearch.common.collect.Map.of("runtime", (name, node, parserContext) -> new Mapper.Builder(name) { - @Override - public Mapper build(ContentPath path) { - return new DummyMapper(name, new DummyMappedFieldType(name)); - } - }); - } - })); - when(mapperService.fieldType("pig")).thenReturn(new DummyMappedFieldType("pig")); - when(mapperService.simpleMatchToFullName("*")).thenReturn(org.elasticsearch.common.collect.Set.of("pig")); + public void testSearchRequestRuntimeFields() { /* * Making these immutable here test that we don't modify them. * Modifying them would cause all kinds of problems if two * shards are parsed on the same node. */ Map runtimeMappings = org.elasticsearch.common.collect.Map.ofEntries( - org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "keyword")), - org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "keyword")) + org.elasticsearch.common.collect.Map.entry("cat", org.elasticsearch.common.collect.Map.of("type", "test")), + org.elasticsearch.common.collect.Map.entry("dog", org.elasticsearch.common.collect.Map.of("type", "test")) ); - QueryShardContext qsc = new QueryShardContext( - 0, - mapperService.getIndexSettings(), - BigArrays.NON_RECYCLING_INSTANCE, - null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - mapperService, - null, - null, - NamedXContentRegistry.EMPTY, - new NamedWriteableRegistry(org.elasticsearch.common.collect.List.of()), + QueryShardContext qsc = createQueryShardContext( + "uuid", null, - null, - () -> 0, - "test", - null, - () -> true, - null, - runtimeMappings - ); + org.elasticsearch.common.collect.Map.of( + "pig", new MockFieldMapper.FakeFieldType("pig"), + "cat", new MockFieldMapper.FakeFieldType("cat")), + runtimeMappings, + Collections.singletonList(new MapperPlugin() { + @Override + public Map getRuntimeFieldTypes() { + return org.elasticsearch.common.collect.Map.of( + "test", (name, node, parserContext) -> new TestRuntimeField(name)); + } + })); assertTrue(qsc.isFieldMapped("cat")); - assertThat(qsc.getFieldType("cat"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("cat"), instanceOf(TestRuntimeField.class)); assertThat(qsc.simpleMatchToIndexNames("cat"), equalTo(org.elasticsearch.common.collect.Set.of("cat"))); assertTrue(qsc.isFieldMapped("dog")); - assertThat(qsc.getFieldType("dog"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("dog"), instanceOf(TestRuntimeField.class)); assertThat(qsc.simpleMatchToIndexNames("dog"), equalTo(org.elasticsearch.common.collect.Set.of("dog"))); assertTrue(qsc.isFieldMapped("pig")); - assertThat(qsc.getFieldType("pig"), instanceOf(DummyMappedFieldType.class)); + assertThat(qsc.getFieldType("pig"), instanceOf(MockFieldMapper.FakeFieldType.class)); assertThat(qsc.simpleMatchToIndexNames("pig"), equalTo(org.elasticsearch.common.collect.Set.of("pig"))); assertThat(qsc.simpleMatchToIndexNames("*"), equalTo(org.elasticsearch.common.collect.Set.of("cat", "dog", "pig"))); } public static QueryShardContext createQueryShardContext(String indexUuid, String clusterAlias) { - return createQueryShardContext(indexUuid, clusterAlias, null); + return createQueryShardContext(indexUuid, clusterAlias, name -> { + throw new UnsupportedOperationException(); + }); } private static QueryShardContext createQueryShardContext( String indexUuid, String clusterAlias, - TriFunction runtimeDocValues + Function fieldTypeLookup ) { - MapperService mapperService = mockMapperService(indexUuid, org.elasticsearch.common.collect.List.of()); - if (runtimeDocValues != null) { - when(mapperService.fieldType(any())).thenAnswer(fieldTypeInv -> { - String fieldName = (String)fieldTypeInv.getArguments()[0]; - return mockFieldType(fieldName, (leafSearchLookup, docId) -> runtimeDocValues.apply(fieldName, leafSearchLookup, docId)); - }); - } + return createQueryShardContext(indexUuid, clusterAlias, new HashMap() { + @Override + public MappedFieldType get(Object key) { + return fieldTypeLookup.apply(key.toString()); + } + }, Collections.emptyMap(), Collections.emptyList()); + } + + private static QueryShardContext createQueryShardContext( + String indexUuid, + String clusterAlias, + Map fieldTypeLookup, + Map runtimeMappings, + List mapperPlugins + ) { + MapperService mapperService = createMapperService(indexUuid, fieldTypeLookup, mapperPlugins); final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), - null, null, () -> nowInMillis, clusterAlias, null, () -> true, null); + null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, runtimeMappings); } - private static MapperService mockMapperService(String indexUuid, List mapperPlugins) { + private static MapperService createMapperService(String indexUuid, + Map fieldTypeLookup, + List mapperPlugins) { IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder("index"); indexMetadataBuilder.settings(Settings.builder().put("index.version.created", Version.CURRENT) .put("index.number_of_shards", 1) @@ -404,60 +403,119 @@ private static MapperService mockMapperService(String indexUuid, List { + throw new UnsupportedOperationException(); + }, () -> true, null) { + @Override + public MappedFieldType fieldType(String name) { + return fieldTypeLookup.get(name); + } - MapperService mapperService = mock(MapperService.class); - when(mapperService.getIndexSettings()).thenReturn(indexSettings); - when(mapperService.index()).thenReturn(indexMetadata.getIndex()); - when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); - Map typeParserMap = IndicesModule.getMappers(mapperPlugins); - Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(name -> null, typeParserMap::get, - Version.CURRENT, () -> null, null, null, mapperService.getIndexAnalyzers(), mapperService.getIndexSettings(), - () -> { + @Override + public Set simpleMatchToFullName(String pattern) { + if (Regex.isMatchAllPattern(pattern)) { + return Collections.unmodifiableSet(fieldTypeLookup.keySet()); + } throw new UnsupportedOperationException(); - }); - when(mapperService.parserContext()).thenReturn(parserContext); - return mapperService; + } + }; } - private static MappedFieldType mockFieldType(String fieldName, BiFunction runtimeDocValues) { - MappedFieldType fieldType = mock(MappedFieldType.class); - when(fieldType.name()).thenReturn(fieldName); - when(fieldType.fielddataBuilder(any(), any())).thenAnswer(builderInv -> { - @SuppressWarnings("unchecked") - Supplier searchLookup = ((Supplier) builderInv.getArguments()[1]); - IndexFieldData indexFieldData = mock(IndexFieldData.class); - when(indexFieldData.load(any())).thenAnswer(loadArgs -> { - LeafReaderContext leafReaderContext = (LeafReaderContext) loadArgs.getArguments()[0]; - LeafFieldData leafFieldData = mock(LeafFieldData.class); - when(leafFieldData.getScriptValues()).thenAnswer(scriptValuesArgs -> new ScriptDocValues() { - String value; + private static Function fieldTypeLookup( + TriFunction runtimeDocValues) { + return name -> new TestRuntimeField(name) { + @Override + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, + Supplier searchLookup) { + return (cache, breakerService) -> new IndexFieldData() { + @Override + public String getFieldName() { + return name; + } + + @Override + public ValuesSourceType getValuesSourceType() { + throw new UnsupportedOperationException(); + } + + @Override + public LeafFieldData load(LeafReaderContext context) { + return new LeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues() { + String value; + + @Override + public int size() { + return 1; + } + + @Override + public String get(int index) { + assert index == 0; + return value; + } + + @Override + public void setNextDocId(int docId) { + assert docId >= 0; + LeafSearchLookup leafLookup = searchLookup.get() + .getLeafSearchLookup(context); + leafLookup.setDocument(docId); + value = runtimeDocValues.apply(name, leafLookup, docId); + } + }; + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + throw new UnsupportedOperationException(); + } + }; + } @Override - public int size() { - return 1; + public LeafFieldData loadDirect(LeafReaderContext context) { + throw new UnsupportedOperationException(); } @Override - public String get(int index) { - assert index == 0; - return value; + public SortField sortField(Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + boolean reverse) { + throw new UnsupportedOperationException(); } @Override - public void setNextDocId(int docId) { - assert docId >= 0; - LeafSearchLookup leafLookup = searchLookup.get().getLeafSearchLookup(leafReaderContext); - leafLookup.setDocument(docId); - value = runtimeDocValues.apply(leafLookup, docId); + public BucketedSort newBucketedSort(BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra) { + throw new UnsupportedOperationException(); } - }); - return leafFieldData; - }); - IndexFieldData.Builder builder = mock(IndexFieldData.Builder.class); - when(builder.build(any(), any())).thenAnswer(buildInv -> indexFieldData); - return builder; - }); - return fieldType; + }; + } + }; } private static List collect(String field, QueryShardContext queryShardContext) throws IOException { @@ -513,46 +571,4 @@ public void collect(int doc) throws IOException { return result; } } - - private static class DummyMapper extends FieldMapper { - protected DummyMapper(String simpleName, MappedFieldType mappedFieldType) { - super(simpleName, mappedFieldType, org.elasticsearch.common.collect.Map.of(), MultiFields.empty(), CopyTo.empty()); - } - - @Override - protected void parseCreateField(ParseContext context) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public Builder getMergeBuilder() { - throw new UnsupportedOperationException(); - } - - @Override - protected String contentType() { - throw new UnsupportedOperationException(); - } - } - - private static class DummyMappedFieldType extends MappedFieldType { - DummyMappedFieldType(String name) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, null); - } - - @Override - public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); - } - - @Override - public String typeName() { - return "runtime"; - } - - @Override - public Query termQuery(Object value, QueryShardContext context) { - throw new UnsupportedOperationException(); - } - } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index ccfdd34aa81f2..e95d401eff356 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; @@ -193,6 +194,19 @@ public Map getMetadataMappers() { assertThat(e.getMessage(), containsString("already registered")); } + public void testDuplicateRuntimeFieldPlugin() { + MapperPlugin plugin = new MapperPlugin() { + @Override + public Map getRuntimeFieldTypes() { + return Collections.singletonMap("test", (name, node, parserContext) -> null); + } + }; + List plugins = Arrays.asList(plugin, plugin); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new IndicesModule(plugins)); + assertThat(e.getMessage(), containsString("already registered")); + } + public void testDuplicateFieldNamesMapper() { List plugins = Arrays.asList(new MapperPlugin() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index b8ac57de07d90..df78993331d6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -172,6 +172,15 @@ protected final MapperService createMapperService(Version version, Settings settings, BooleanSupplier idFieldDataEnabled, XContentBuilder mapping) throws IOException { + + MapperService mapperService = createMapperService(version, settings, idFieldDataEnabled); + merge(mapperService, mapping); + return mapperService; + } + + protected final MapperService createMapperService(Version version, + Settings settings, + BooleanSupplier idFieldDataEnabled) { settings = Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", 1) @@ -191,7 +200,7 @@ protected final MapperService createMapperService(Version version, ); ScriptService scriptService = new ScriptService(getIndexSettings(), scriptModule.engines, scriptModule.contexts); SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap()); - MapperService mapperService = new MapperService( + return new MapperService( indexSettings, createIndexAnalyzers(indexSettings), xContentRegistry(), @@ -201,8 +210,6 @@ protected final MapperService createMapperService(Version version, idFieldDataEnabled, scriptService ); - merge(mapperService, mapping); - return mapperService; } protected final void withLuceneIndex( @@ -290,6 +297,20 @@ protected final XContentBuilder fieldMapping(CheckedConsumer buildField) throws IOException { + return runtimeMapping(b -> { + b.startObject("field"); + buildField.accept(b); + b.endObject(); + }); + } + + protected final XContentBuilder runtimeMapping(CheckedConsumer buildFields) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("runtime"); + buildFields.accept(builder); + return builder.endObject().endObject().endObject(); + } + private AggregationContext aggregationContext( ValuesSourceRegistry valuesSourceRegistry, MapperService mapperService, diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 78c3330b23aa1..08bfae61d0cca 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,9 +19,6 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -30,9 +27,6 @@ // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { - static Settings DEFAULT_SETTINGS = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id) - .build(); public MockFieldMapper(String fullName) { this(new FakeFieldType(fullName)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index a87df96e4556a..275a5699cef03 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -789,7 +789,7 @@ private void writeTestDoc(MappedFieldType fieldType, String fieldName, RandomInd private static class MockParserContext extends Mapper.TypeParser.ParserContext { MockParserContext() { - super(null, null, Version.CURRENT, null, null, null, null, null, null); + super(null, null, null, Version.CURRENT, null, null, null, null, null, null); } @Override diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java index 1083f4d246ac4..3279ca35b3581 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlattenedFieldLookupTests.java @@ -39,7 +39,7 @@ public void testFieldTypeLookup() { String fieldName = "object1.object2.field"; FlattenedFieldMapper mapper = createFlattenedMapper(fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), emptyList(), emptyList()); assertEquals(mapper.fieldType(), lookup.get(fieldName)); String objectKey = "key1.key2"; @@ -60,7 +60,7 @@ public void testFieldTypeLookupWithAlias() { String aliasName = "alias"; FieldAliasMapper alias = new FieldAliasMapper(aliasName, aliasName, fieldName); - FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias)); + FieldTypeLookup lookup = new FieldTypeLookup(singletonList(mapper), singletonList(alias), emptyList()); assertEquals(mapper.fieldType(), lookup.get(aliasName)); String objectKey = "key1.key2"; @@ -83,11 +83,11 @@ public void testFieldTypeLookupWithMultipleFields() { FlattenedFieldMapper mapper2 = createFlattenedMapper(field2); FlattenedFieldMapper mapper3 = createFlattenedMapper(field3); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); - lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList()); + lookup = new FieldTypeLookup(Arrays.asList(mapper1, mapper2, mapper3), emptyList(), emptyList()); assertNotNull(lookup.get(field1 + ".some.key")); assertNotNull(lookup.get(field2 + ".some.key")); assertNotNull(lookup.get(field3 + ".some.key")); @@ -124,7 +124,7 @@ public void testFieldLookupIterator() { MockFieldMapper mapper = new MockFieldMapper("foo"); FlattenedFieldMapper flattenedMapper = createFlattenedMapper("object1.object2.field"); - FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList()); + FieldTypeLookup lookup = new FieldTypeLookup(Arrays.asList(mapper, flattenedMapper), emptyList(), emptyList()); Set fieldNames = new HashSet<>(); lookup.filter(ft -> true).forEach(ft -> fieldNames.add(ft.name())); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 09d430222c871..24d985ff120a1 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -118,6 +118,12 @@ private void addAirlineData() throws IOException { Request createAirlineDataRequest = new Request("PUT", "/airline-data"); createAirlineDataRequest.setJsonEntity("{" + " \"mappings\": {" + + " \"runtime\": {" + + " \"airline_lowercase_rt\": { " + + " \"type\":\"keyword\"," + + " \"script\" : { \"source\": \"emit(params._source.airline.toLowerCase())\" }" + + " }" + + " }," + " \"properties\": {" + " \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional + " \"airline\": {" @@ -127,11 +133,6 @@ private void addAirlineData() throws IOException { + " \"keyword\":{\"type\":\"keyword\"}" + " }" + " }," - + " \"airline_lowercase_rt\": { " - + " \"type\":\"runtime\"," - + " \"runtime_type\": \"keyword\"," - + " \"script\" : { \"source\": \"emit(params._source.airline.toLowerCase())\" }" - + " }," + " \"responsetime\": { \"type\":\"float\"}" + " }" + " }" diff --git a/x-pack/plugin/runtime-fields/qa/build.gradle b/x-pack/plugin/runtime-fields/qa/build.gradle index 1ab8b6efe9250..9fd5e0e62ecf7 100644 --- a/x-pack/plugin/runtime-fields/qa/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/build.gradle @@ -1,4 +1,4 @@ -// Shared infratructure +// Shared infrastructure apply plugin: 'elasticsearch.build' @@ -43,6 +43,8 @@ subprojects { 'suggest', ] if (project.name.equals('core-with-mapped')) { + //disabled until runtime fields can be created from a dynamic template + yamlRestTest.enabled = false suites += [ // These two don't support runtime fields on the request. Should they? 'field_caps', @@ -76,4 +78,4 @@ subprojects { ].join(',') } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java index 0c19187c088d0..131e124bac48d 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-mapped/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/mapped/CoreWithMappedRuntimeFieldsIT.java @@ -8,14 +8,12 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; import org.elasticsearch.xpack.runtimefields.test.CoreTestTranslater; -import java.util.HashMap; import java.util.Map; /** @@ -43,20 +41,8 @@ protected Map dynamicTemplateFor(String type) { protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { @Override - protected boolean modifyMappingProperties(String index, Map properties) { - Map newProperties = new HashMap<>(properties.size()); - Map> runtimeProperties = new HashMap<>(properties.size()); - if (false == runtimeifyMappingProperties(properties, newProperties, runtimeProperties)) { - return false; - } - for (Map.Entry> runtimeProperty : runtimeProperties.entrySet()) { - runtimeProperty.getValue().put("runtime_type", runtimeProperty.getValue().get("type")); - runtimeProperty.getValue().put("type", "runtime"); - newProperties.put(runtimeProperty.getKey(), runtimeProperty.getValue()); - } - properties.clear(); - properties.putAll(newProperties); - return true; + protected boolean modifyMappingProperties(String index, Map properties, Map runtimeFields) { + return runtimeifyMappingProperties(properties, runtimeFields); } @Override diff --git a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java index 69325a5729dd0..302888349a4ae 100644 --- a/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java +++ b/x-pack/plugin/runtime-fields/qa/core-with-search/src/yamlRestTest/java/org/elasticsearch/xpack/runtimefields/test/search/CoreTestsWithSearchRuntimeFieldsIT.java @@ -8,7 +8,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; @@ -57,9 +56,9 @@ protected Map dynamicTemplateFor(String type) { @Override protected Suite suite(ClientYamlTestCandidate candidate) { return new Suite(candidate) { - private Map>> runtimeMappingsAfterSetup; + private Map> runtimeMappingsAfterSetup; private Map> mappedFieldsAfterSetup; - private Map>> runtimeMappings; + private Map> runtimeMappings; private Map> mappedFields; @Override @@ -83,16 +82,12 @@ public boolean modifySections(List executables) { } @Override - protected boolean modifyMappingProperties(String index, Map properties) { - Map untouchedMapping = new HashMap<>(); - Map> runtimeMapping = new HashMap<>(); - if (false == runtimeifyMappingProperties(properties, untouchedMapping, runtimeMapping)) { + protected boolean modifyMappingProperties(String index, Map properties, Map runtimeFields) { + if (false == runtimeifyMappingProperties(properties, runtimeFields)) { return false; } - properties.clear(); - properties.putAll(untouchedMapping); - mappedFields.put(index, untouchedMapping.keySet()); - runtimeMappings.put(index, runtimeMapping); + mappedFields.put(index, properties.keySet()); + runtimeMappings.put(index, runtimeFields); return true; } @@ -116,6 +111,7 @@ protected boolean modifySearch(ApiCallSection search) { return mergeMappings(new String[] { "*" }); } String[] patterns = Arrays.stream(index.split(",")).map(m -> m.equals("_all") ? "*" : m).toArray(String[]::new); + // TODO this is always false? if (patterns.length == 0 && Regex.isSimpleMatchPattern(patterns[0])) { return runtimeMappings.get(patterns[0]); } @@ -123,13 +119,14 @@ protected boolean modifySearch(ApiCallSection search) { } private Map mergeMappings(String[] patterns) { - Map> merged = new HashMap<>(); - for (Map.Entry>> indexEntry : runtimeMappings.entrySet()) { + Map merged = new HashMap<>(); + for (Map.Entry> indexEntry : runtimeMappings.entrySet()) { if (false == Regex.simpleMatch(patterns, indexEntry.getKey())) { continue; } - for (Map.Entry> field : indexEntry.getValue().entrySet()) { - Map mergedConfig = merged.get(field.getKey()); + for (Map.Entry field : indexEntry.getValue().entrySet()) { + @SuppressWarnings("unchecked") + Map mergedConfig = (Map) merged.get(field.getKey()); if (mergedConfig == null) { merged.put(field.getKey(), field.getValue()); } else if (false == mergedConfig.equals(field.getValue())) { @@ -164,10 +161,7 @@ protected boolean handleIndex(IndexRequest index) { return false; } Map map = XContentHelper.convertToMap(index.source(), false, index.getContentType()).v2(); - Map> indexRuntimeMappings = runtimeMappings.computeIfAbsent( - index.index(), - i -> new HashMap<>() - ); + Map indexRuntimeMappings = runtimeMappings.computeIfAbsent(index.index(), i -> new HashMap<>()); Set indexMappedfields = mappedFields.computeIfAbsent( index.index(), i -> org.elasticsearch.common.collect.Set.of() diff --git a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java index b034b572c7e6e..6fc52cb49510d 100644 --- a/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java +++ b/x-pack/plugin/runtime-fields/qa/src/main/java/org/elasticsearch/xpack/runtimefields/test/CoreTestTranslater.java @@ -116,6 +116,7 @@ protected static Map dynamicTemplateToDisableRuntimeCompatibleFi return org.elasticsearch.common.collect.Map.of("type", type, "index", false, "doc_values", false); } + // TODO there isn't yet a way to create fields in the runtime section from a dynamic template protected static Map dynamicTemplateToAddRuntimeFields(String type) { return org.elasticsearch.common.collect.Map.ofEntries( org.elasticsearch.common.collect.Map.entry("type", "runtime"), @@ -280,25 +281,25 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { Object settings = body.get("settings"); if (settings instanceof Map && ((Map) settings).containsKey("sort.field")) { /* - * You can't sort the index on a runtime_keyword and it is - * hard to figure out if the sort was a runtime_keyword so - * let's just skip this test. + * You can't sort the index on a runtime field */ continue; } - Object mapping = body.get("mappings"); - if (false == (mapping instanceof Map)) { + @SuppressWarnings("unchecked") + Map mapping = (Map) body.get("mappings"); + if (mapping == null) { continue; } - Object properties = ((Map) mapping).get("properties"); - if (false == (properties instanceof Map)) { + @SuppressWarnings("unchecked") + Map propertiesMap = (Map) ((Map) mapping).get("properties"); + if (propertiesMap == null) { continue; } - @SuppressWarnings("unchecked") - Map propertiesMap = (Map) properties; - if (false == modifyMappingProperties(index, propertiesMap)) { + Map runtimeFields = new HashMap<>(); + if (false == modifyMappingProperties(index, propertiesMap, runtimeFields)) { return false; } + mapping.put("runtime", runtimeFields); } return true; } @@ -306,26 +307,21 @@ private boolean modifyCreateIndex(ApiCallSection createIndex) { /** * Modify the mapping defined in the test. */ - protected abstract boolean modifyMappingProperties(String index, Map properties); + protected abstract boolean modifyMappingProperties(String index, Map mappings, Map runtimeFields); /** * Modify the provided map in place, translating all fields into * runtime fields that load from source. * @return true if this mapping supports runtime fields, false otherwise */ - protected final boolean runtimeifyMappingProperties( - Map properties, - Map untouchedProperties, - Map> runtimeProperties - ) { + protected final boolean runtimeifyMappingProperties(Map properties, Map runtimeFields) { for (Map.Entry property : properties.entrySet()) { if (false == property.getValue() instanceof Map) { - untouchedProperties.put(property.getKey(), property.getValue()); continue; } @SuppressWarnings("unchecked") Map propertyMap = (Map) property.getValue(); - String name = property.getKey().toString(); + String name = property.getKey(); String type = Objects.toString(propertyMap.get("type")); if ("nested".equals(type)) { // Our loading scripts can't be made to manage nested fields so we have to skip those tests. @@ -333,40 +329,39 @@ protected final boolean runtimeifyMappingProperties( } if ("false".equals(Objects.toString(propertyMap.get("doc_values")))) { // If doc_values is false we can't emulate with scripts. So we keep the old definition. `null` and `true` are fine. - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("false".equals(Objects.toString(propertyMap.get("index")))) { // If index is false we can't emulate with scripts - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if ("true".equals(Objects.toString(propertyMap.get("store")))) { // If store is true we can't emulate with scripts - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_above")) { // Scripts don't support ignore_above so we skip those fields - untouchedProperties.put(property.getKey(), property.getValue()); continue; } if (propertyMap.containsKey("ignore_malformed")) { // Our source reading script doesn't emulate ignore_malformed - untouchedProperties.put(property.getKey(), property.getValue()); continue; } String toLoad = painlessToLoadFromSource(name, type); if (toLoad == null) { - untouchedProperties.put(property.getKey(), property.getValue()); continue; } Map runtimeConfig = new HashMap<>(propertyMap); + runtimeConfig.put("type", type); runtimeConfig.put("script", toLoad); runtimeConfig.remove("store"); runtimeConfig.remove("index"); runtimeConfig.remove("doc_values"); - runtimeProperties.put(property.getKey(), runtimeConfig); + runtimeFields.put(name, runtimeConfig); + + // we disable the mapped fields and shadow them with their corresponding runtime field + propertyMap.put("doc_values", false); + propertyMap.put("index", false); } /* * Its tempting to return false here if we didn't make any runtime diff --git a/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java b/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java index 6c26fb4730929..a12439f9925f1 100644 --- a/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java +++ b/x-pack/plugin/runtime-fields/qa/with-security/src/javaRestTest/java/org/elasticsearch/xpack/security/PermissionsIT.java @@ -68,13 +68,14 @@ public void testDLS() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"year\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"keyword\",\n" + + " \"type\" : \"keyword\", \n" + " \"script\" : \"emit(doc['date'].value.substring(0,4))\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" @@ -110,13 +111,14 @@ public void testFLSProtectsData() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"hidden\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"hidden_values_count\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"long\",\n" + + " \"type\" : \"long\", \n" + " \"script\" : \"emit(doc['hidden'].size())\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"hidden\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" @@ -159,13 +161,14 @@ public void testFLSOnRuntimeField() throws IOException { createIndex.setJsonEntity( "{\n" + " \"mappings\" : {\n" - + " \"properties\" : {\n" - + " \"date\" : {\"type\" : \"keyword\"},\n" + + " \"runtime\" : {\n" + " \"year\" : {\n" - + " \"type\" : \"runtime\", \n" - + " \"runtime_type\" : \"keyword\",\n" + + " \"type\" : \"keyword\", \n" + " \"script\" : \"emit(doc['date'].value.substring(0,4))\"\n" + " }\n" + + " },\n" + + " \"properties\" : {\n" + + " \"date\" : {\"type\" : \"keyword\"}\n" + " }\n" + " }\n" + "}\n" diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 45c08120c400a..679c3724f92b9 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -6,29 +6,48 @@ package org.elasticsearch.xpack.runtimefields; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.BooleanFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.xpack.runtimefields.mapper.BooleanFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.BooleanScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DateFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DateScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.DoubleFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.DoubleScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.GeoPointFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.GeoPointScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.IpFieldScript; +import org.elasticsearch.xpack.runtimefields.mapper.IpScriptFieldType; +import org.elasticsearch.xpack.runtimefields.mapper.KeywordScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.LongFieldScript; -import org.elasticsearch.xpack.runtimefields.mapper.RuntimeFieldMapper; +import org.elasticsearch.xpack.runtimefields.mapper.LongScriptFieldType; import org.elasticsearch.xpack.runtimefields.mapper.StringFieldScript; -import java.util.Collections; import java.util.List; import java.util.Map; public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { @Override - public Map getMappers() { - return Collections.singletonMap(RuntimeFieldMapper.CONTENT_TYPE, RuntimeFieldMapper.PARSER); + public Map getRuntimeFieldTypes() { + return org.elasticsearch.common.collect.Map.ofEntries( + org.elasticsearch.common.collect.Map.entry(BooleanFieldMapper.CONTENT_TYPE, BooleanScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(NumberFieldMapper.NumberType.LONG.typeName(), LongScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(NumberFieldMapper.NumberType.DOUBLE.typeName(), DoubleScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(IpFieldMapper.CONTENT_TYPE, IpScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(DateFieldMapper.CONTENT_TYPE, DateScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(KeywordFieldMapper.CONTENT_TYPE, KeywordScriptFieldType.PARSER), + org.elasticsearch.common.collect.Map.entry(GeoPointFieldMapper.CONTENT_TYPE, GeoPointScriptFieldType.PARSER) + ); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java index 0feb612f82018..5b5ca86fff2b8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptFieldType.java @@ -12,52 +12,58 @@ import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.DocValueFetcher; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.RuntimeFieldType; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.time.ZoneId; +import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; /** * Abstract base {@linkplain MappedFieldType} for scripted fields. */ -abstract class AbstractScriptFieldType extends MappedFieldType { +abstract class AbstractScriptFieldType extends RuntimeFieldType { protected final Script script; private final TriFunction, SearchLookup, LeafFactory> factory; + private final CheckedBiConsumer toXContent; + + AbstractScriptFieldType(String name, TriFunction, SearchLookup, LeafFactory> factory, Builder builder) { + this(name, factory, builder.script.getValue(), builder.meta.getValue(), builder::toXContent); + } AbstractScriptFieldType( String name, - Script script, TriFunction, SearchLookup, LeafFactory> factory, - Map meta + Script script, + Map meta, + CheckedBiConsumer toXContent ) { - super(name, false, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); - this.script = script; + super(name, meta); this.factory = factory; - } - - protected abstract String runtimeType(); - - @Override - public final String typeName() { - return RuntimeFieldMapper.CONTENT_TYPE; - } - - @Override - public final String familyTypeName() { - return runtimeType(); + this.script = script; + this.toXContent = toXContent; } @Override @@ -101,8 +107,8 @@ public final Query rangeQuery( QueryShardContext context ) { if (relation == ShapeRelation.DISJOINT) { - String message = "Field [%s] of type [%s] with runtime type [%s] does not support DISJOINT ranges"; - throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + String message = "Runtime field [%s] of type [%s] does not support DISJOINT ranges"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName())); } return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context); } @@ -174,46 +180,115 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew private String unsupported(String query, String supported) { return String.format( Locale.ROOT, - "Can only use %s queries on %s fields - not on [%s] which is of type [%s] with runtime_type [%s]", + "Can only use %s queries on %s fields - not on [%s] which is a runtime field of type [%s]", query, supported, name(), - RuntimeFieldMapper.CONTENT_TYPE, - runtimeType() + typeName() ); } protected final void checkAllowExpensiveQueries(QueryShardContext context) { if (context.allowExpensiveQueries() == false) { throw new ElasticsearchException( - "queries cannot be executed against [" - + RuntimeFieldMapper.CONTENT_TYPE - + "] fields while [" - + ALLOW_EXPENSIVE_QUERIES.getKey() - + "] is set to [false]." + "queries cannot be executed against runtime fields while [" + ALLOW_EXPENSIVE_QUERIES.getKey() + "] is set to [false]." ); } } - /** - * The format that this field should use. The default implementation is - * {@code null} because most fields don't support formats. - */ - protected String format() { - return null; + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup lookup, String format) { + return new DocValueFetcher(docValueFormat(format, null), lookup.doc().getForField(this)); + } + + @Override + protected final void doXContentBody(XContentBuilder builder, boolean includeDefaults) throws IOException { + toXContent.accept(builder, includeDefaults); } /** - * The locale that this field's format should use. The default - * implementation is {@code null} because most fields don't - * support formats. + * For runtime fields the {@link RuntimeFieldType.Parser} returns directly the {@link MappedFieldType}. + * Internally we still create a {@link Builder} so we reuse the {@link FieldMapper.Parameter} infrastructure, + * but {@link Builder#init(FieldMapper)} and {@link Builder#build(ContentPath)} are never called as + * {@link RuntimeFieldTypeParser#parse(String, Map, Mapper.TypeParser.ParserContext)} calls + * {@link Builder#parse(String, Mapper.TypeParser.ParserContext, Map)} and returns the corresponding + * {@link MappedFieldType}. */ - protected Locale formatLocale() { - return null; + abstract static class Builder extends FieldMapper.Builder { + final FieldMapper.Parameter> meta = FieldMapper.Parameter.metaParam(); + final FieldMapper.Parameter