WIP: Support unmapped fields in search 'fields' option

rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -295,3 +295,111 @@ setup:
   - is_true: hits.hits.0._id
   - match: { hits.hits.0.fields.count: [2] }
   - is_false: hits.hits.0.fields.count_without_dv
+---
+Test unmapped field:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support isn't yet backported
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 dynamic: false
+                 properties:
+                    f1:
+                       type: keyword
+                    f2:
+                       type: object
+                       enabled: false
+                    f3:
+                       type: object
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: some text
+              f2:
+                 a: foo
+                 b: bar
+              f3:
+                 c: baz
+              f4: some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - f1
+              - { "field" : "f4", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+  -  do:
+        search:
+           index: test
+           body:
+              fields:
+              - { "field" : "f*", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+  -  match:
+        hits.hits.0.fields.f2\.a:
+        - foo
+  -  match:
+        hits.hits.0.fields.f2\.b:
+        - bar
+  -  match:
+        hits.hits.0.fields.f3\.c:
+        - baz
+  -  match:
+        hits.hits.0.fields.f4:
+        - some other text
+---
+Test unmapped fields inside disabled objects:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support isn't yet backported
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                    f1:
+                       type: object
+                       enabled: false
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              f1: 
+                 - some text
+                 - a: b
+                 -
+                   - 1
+                   - 2
+                   - 3
+  -  do:
+        search:
+           index: test
+           body:
+              fields: [ { "field" : "*", "include_unmapped" : true } ]
+  -  match:
+        hits.hits.0.fields.f1: 
+        - some text
+        -
+          - 1
+          - 2
+          - 3
+  -  match:
+        hits.hits.0.fields.f1\.a:
+        - b

server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java

@@ -138,7 +138,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
             }
         }
         if (options.getFetchFields() != null) {
-            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
+            options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format, ff.includeUnmapped.orElse(null)));
         }
         if (options.getDocValueFields() != null) {
             options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));

server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java

@@ -310,7 +310,7 @@ public SearchRequestBuilder addDocValueField(String name) {
      * @param name The field to load
      */
     public SearchRequestBuilder addFetchField(String name) {
-        sourceBuilder().fetchField(name, null);
+        sourceBuilder().fetchField(name, null, null);
         return this;
     }
 
@@ -319,9 +319,10 @@ public SearchRequestBuilder addFetchField(String name) {
      *
      * @param name The field to load
      * @param format an optional format string used when formatting values, for example a date format.
+     * @param includeUnmapped whether this field pattern should also include unmapped fields
      */
-    public SearchRequestBuilder addFetchField(String name, String format) {
-        sourceBuilder().fetchField(name, format);
+    public SearchRequestBuilder addFetchField(String name, String format, boolean includeUnmapped) {
+        sourceBuilder().fetchField(name, format, null);
         return this;
     }
 

server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java

@@ -47,6 +47,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT;
@@ -395,10 +396,20 @@ public InnerHitBuilder addFetchField(String name) {
      * @param format an optional format string used when formatting values, for example a date format.
      */
     public InnerHitBuilder addFetchField(String name, @Nullable String format) {
+        return addFetchField(name, format, false);
+    }
+
+    /**
+     * Adds a field to load and return as part of the search request.
+     * @param name the field name.
+     * @param format an optional format string used when formatting values, for example a date format.
+     * @param includeUnmapped whether unmapped fields should be returned as well
+     */
+    public InnerHitBuilder addFetchField(String name, @Nullable String format, boolean includeUnmapped) {
         if (fetchFields == null || fetchFields.isEmpty()) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(new FieldAndFormat(name, format, Optional.of(includeUnmapped)));
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java

@@ -446,17 +446,24 @@ public List<FieldAndFormat> docValueFields() {
     /**
      * Adds a field to load and return as part of the search request.
      */
-    public TopHitsAggregationBuilder fetchField(String field, String format) {
+    public TopHitsAggregationBuilder fetchField(String field, String format, boolean includeUnmapped) {
         if (field == null) {
             throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
         }
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(field, format));
+        fetchFields.add(new FieldAndFormat(field, format, Optional.of(includeUnmapped)));
         return this;
     }
 
+    /**
+     * Adds a field to load and return as part of the search request.
+     */
+    public TopHitsAggregationBuilder fetchField(String field, String format) {
+        return fetchField(field, format, false);
+    }
+
     /**
      * Adds a field to load and return as part of the search request.
      */

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -65,6 +65,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
@@ -868,19 +869,19 @@ public List<FieldAndFormat> fetchFields() {
      * Adds a field to load and return as part of the search request.
      */
     public SearchSourceBuilder fetchField(String name) {
-        return fetchField(name, null);
+        return fetchField(name, null, null);
     }
 
     /**
      * Adds a field to load and return as part of the search request.
      * @param name the field name.
      * @param format an optional format string used when formatting values, for example a date format.
      */
-    public SearchSourceBuilder fetchField(String name, @Nullable String format) {
+    public SearchSourceBuilder fetchField(String name, @Nullable String format, @Nullable Boolean includeUnmapped) {
         if (fetchFields == null) {
             fetchFields = new ArrayList<>();
         }
-        fetchFields.add(new FieldAndFormat(name, format));
+        fetchFields.add(new FieldAndFormat(name, format, Optional.ofNullable(includeUnmapped)));
         return this;
     }
 

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesContext.java

@@ -45,7 +45,7 @@ public FetchDocValuesContext(QueryShardContext shardContext, List<FieldAndFormat
             Collection<String> fieldNames = shardContext.simpleMatchToIndexNames(field.field);
             for (String fieldName : fieldNames) {
                 if (shardContext.isFieldMapped(fieldName)) {
-                    fields.add(new FieldAndFormat(fieldName, field.format));
+                    fields.add(new FieldAndFormat(fieldName, field.format, field.includeUnmapped));
                 }
             }
         }

server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java

@@ -54,6 +54,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext fetchContext) {
         }
 
         FieldFetcher fieldFetcher = FieldFetcher.create(fetchContext.getQueryShardContext(), searchLookup, fetchFieldsContext.fields());
+
         return new FetchSubPhaseProcessor() {
             @Override
             public void setNextReader(LeafReaderContext readerContext) {

server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldAndFormat.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.fetch.subphase;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -32,6 +33,7 @@
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Wrapper around a field name and the format that should be used to
@@ -40,14 +42,16 @@
 public final class FieldAndFormat implements Writeable, ToXContentObject {
     private static final ParseField FIELD_FIELD = new ParseField("field");
     private static final ParseField FORMAT_FIELD = new ParseField("format");
+    private static final ParseField INCLUDE_UNMAPPED_FIELD = new ParseField("include_unmapped");
 
     private static final ConstructingObjectParser<FieldAndFormat, Void> PARSER =
         new ConstructingObjectParser<>("fetch_field_and_format",
-        a -> new FieldAndFormat((String) a[0], (String) a[1]));
+        a -> new FieldAndFormat((String) a[0], (String) a[1], Optional.ofNullable((Boolean) a[2])));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD_FIELD);
         PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FORMAT_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), INCLUDE_UNMAPPED_FIELD);
     }
 
     /**
@@ -69,6 +73,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (format != null) {
             builder.field(FORMAT_FIELD.getPreferredName(), format);
         }
+        if (this.includeUnmapped.isPresent()) {
+            builder.field(INCLUDE_UNMAPPED_FIELD.getPreferredName(), includeUnmapped);
+        }
         builder.endObject();
         return builder;
     }
@@ -79,28 +86,44 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
     /** The format of the field, or {@code null} if defaults should be used. */
     public final String format;
 
-    /** Sole constructor. */
+    /** Whether to include unmapped fields or not. */
+    public final Optional<Boolean> includeUnmapped;
+
     public FieldAndFormat(String field, @Nullable String format) {
+        this(field, format, Optional.empty());
+    }
+
+    public FieldAndFormat(String field, @Nullable String format, Optional<Boolean> includeUnmapped) {
         this.field = Objects.requireNonNull(field);
         this.format = format;
+        this.includeUnmapped = includeUnmapped;
     }
 
     /** Serialization constructor. */
     public FieldAndFormat(StreamInput in) throws IOException {
         this.field = in.readString();
         format = in.readOptionalString();
+        if (in.getVersion().onOrAfter(Version.CURRENT)) {
+            this.includeUnmapped = Optional.ofNullable(in.readOptionalBoolean());
+        } else {
+            this.includeUnmapped = Optional.empty();
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(field);
         out.writeOptionalString(format);
+        if (out.getVersion().onOrAfter(Version.CURRENT)) {
+            out.writeOptionalBoolean(this.includeUnmapped.orElse(null));
+        }
     }
 
     @Override
     public int hashCode() {
         int h = field.hashCode();
         h = 31 * h + Objects.hashCode(format);
+        h = 31 * h + Boolean.hashCode(this.includeUnmapped.orElse(false));
         return h;
     }
 
@@ -110,6 +133,6 @@ public boolean equals(Object obj) {
             return false;
         }
         FieldAndFormat other = (FieldAndFormat) obj;
-        return field.equals(other.field) && Objects.equals(format, other.format);
+        return field.equals(other.field) && Objects.equals(format, other.format) && Objects.equals(includeUnmapped,other.includeUnmapped);
     }
 }