Add deprecation logging when _all is enabled (#26228)

* Add deprecation logging when _all is enabled

In 6.0 `_all` is not configurable and disabled by default.

Relates to #22144

* Log deprecation regardless of whether _all is enabled or disabled

* Add expected warnings in REST tests

* Assert warnings in unit tests

* Add warning check to various documentation tests

* Add assertWarnings if randomBoolean() is true for enabled

core/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java

@@ -24,6 +24,8 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.lucene.all.AllField;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
@@ -47,6 +49,8 @@
  */
 public class AllFieldMapper extends MetadataFieldMapper {
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(Loggers.getLogger(AllFieldMapper.class));
+
     public static final String NAME = "_all";
 
     public static final String CONTENT_TYPE = "_all";
@@ -135,6 +139,7 @@ public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> n
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
+                    deprecationLogger.deprecated("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
                     boolean enabled = TypeParsers.nodeBooleanValue(name, "enabled", fieldNode);
                     builder.enabled(enabled ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();

core/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java

@@ -92,7 +92,8 @@ public void testSimpleAllMappers() throws Exception {
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllMappersNoBoost() throws Exception {
@@ -114,7 +115,8 @@ public void testAllMappersNoBoost() throws Exception {
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllMappersTermQuery() throws Exception {
@@ -135,7 +137,8 @@ public void testAllMappersTermQuery() throws Exception {
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     // #6187: make sure we see AllTermQuery even when offsets are indexed in the _all field:
@@ -156,7 +159,8 @@ public void testAllMappersWithOffsetsTermQuery() throws Exception {
         assertThat(mapper.fieldType().queryStringTermQuery(new Term("_all", "foobar")),
             Matchers.instanceOf(AllTermQuery.class));
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     // #6187: if _all doesn't index positions then we never use AllTokenStream, even if some fields have boost
@@ -175,7 +179,8 @@ public void testBoostWithOmitPositions() throws Exception {
                 Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
         }
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     // #6187: if no fields were boosted, we shouldn't use AllTokenStream
@@ -193,7 +198,8 @@ public void testNoBoost() throws Exception {
                 Matchers.not(Matchers.instanceOf(AllTokenStream.class)));
         }
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testSimpleAllMappersWithReparse() throws Exception {
@@ -221,7 +227,8 @@ public void testSimpleAllMappersWithReparse() throws Exception {
             }
         }
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testSimpleAllMappersWithStore() throws Exception {
@@ -238,7 +245,8 @@ public void testSimpleAllMappersWithStore() throws Exception {
             assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
         }
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
 
@@ -260,7 +268,8 @@ public void testSimpleAllMappersWithReparseWithStore() throws Exception {
             assertThat(fields[i].fieldType().omitNorms(), equalTo(false));
         }
         assertWarnings("field [include_in_all] is deprecated, as [_all] is deprecated, and will be disallowed" +
-                        " in 6.0, use [copy_to] instead.");
+                        " in 6.0, use [copy_to] instead.",
+                "The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testRandom() throws Exception {
@@ -276,6 +285,7 @@ public void testRandom() throws Exception {
         mappingBuilder.startObject().startObject("test");
         List<Tuple<String, Boolean>> booleanOptionList = new ArrayList<>();
         boolean allDefault = true;
+        boolean allSet = false;
         if (frequently()) {
             allDefault = false;
             mappingBuilder.startObject("_all");
@@ -289,6 +299,7 @@ public void testRandom() throws Exception {
                 booleanOptionList.add(new Tuple<>("store_term_vectors", tv_stored = randomBoolean()));
             }
             if (randomBoolean()) {
+                allSet = true;
                 booleanOptionList.add(new Tuple<>("enabled", enabled = randomBoolean()));
             }
             if (randomBoolean()) {
@@ -352,7 +363,9 @@ public void testRandom() throws Exception {
             xContentBuilder.flush();
             assertThat(bytesStreamOutput.size(), equalTo(0));
         }
-
+        if (allSet) {
+            assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
+        }
     }
 
     public void testMultiField_includeInAllSetToFalse() throws IOException {

core/src/test/java/org/elasticsearch/index/mapper/DocumentMapperParserTests.java

@@ -39,6 +39,7 @@ public void testTypeLevel() throws Exception {
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertThat(mapper.type(), equalTo("type"));
         assertThat(mapper.allFieldMapper().enabled(), equalTo(false));
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testFieldNameWithDots() throws Exception {

core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java

@@ -260,6 +260,7 @@ public void testAllEnabled() throws Exception {
         indexService.mapperService().merge("other_type", disabledAll,
                 MergeReason.MAPPING_UPDATE, random().nextBoolean());
         assertTrue(indexService.mapperService().allEnabled()); // this returns true if any of the types has _all enabled
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
      public void testPartitionedConstraints() {

core/src/test/java/org/elasticsearch/index/mapper/UpdateMappingTests.java

@@ -48,32 +48,37 @@ public void testAllEnabledAfterDisabled() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
         XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
         testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllDisabledAfterEnabled() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
         XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
         testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllDisabledAfterDefaultEnabled() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("some_text").field("type", "text").endObject().endObject().endObject();
         XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
         testConflictWhileMergingAndMappingUnchanged(mapping, mappingUpdate);
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllEnabledAfterEnabled() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().endObject();
         XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
         XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", true).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject();
         testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     public void testAllDisabledAfterDisabled() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().endObject();
         XContentBuilder mappingUpdate = XContentFactory.jsonBuilder().startObject().startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject();
         XContentBuilder expectedMapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_all").field("enabled", false).endObject().startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject();
         testNoConflictWhileMergingAndMappingChanged(mapping, mappingUpdate, expectedMapping);
+        assertWarnings("The [_all] field is deprecated and will be removed in Elasticsearch 6.0");
     }
 
     private void testNoConflictWhileMergingAndMappingChanged(XContentBuilder mapping, XContentBuilder mappingUpdate, XContentBuilder expectedMapping) throws IOException {

docs/reference/mapping.asciidoc

@@ -180,6 +180,7 @@ PUT my_index <1>
 }
 ---------------------------------------
 // CONSOLE
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 <1> Create an index called `my_index`.
 <2> Add mapping types called `user` and `blogpost`.
 <3> Disable the `_all` <<mapping-fields,meta field>> for the `user` mapping type.

docs/reference/mapping/dynamic/default-mapping.asciidoc

@@ -28,6 +28,7 @@ PUT my_index
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 <1> The `_default_` mapping defaults the <<mapping-all-field,`_all`>> field to disabled.
 <2> The `user` type inherits the settings from `_default_`.
 <3> The `blogpost` type overrides the defaults and enables the <<mapping-all-field,`_all`>> field.
@@ -80,6 +81,7 @@ PUT logs-2015.10.01/event/1
 { "message": "error:16" }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 <1> The `logging` template will match any indices beginning with `logs-`.
 <2> Matching indices will be created with a single primary shard.
 <3> The `_all` field will be disabled by default for new type mappings.

docs/reference/mapping/dynamic/templates.asciidoc

@@ -423,6 +423,7 @@ PUT _template/disable_all_field
 }
 --------------------------------------------------
 // CONSOLE
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 <1> Applies the mappings to an `index` which matches the pattern `*`, in other
     words, all new indices.
 <2> Defines the `_default_` type mapping types within the index.

docs/reference/mapping/fields/all-field.asciidoc

@@ -1,6 +1,8 @@
 [[mapping-all-field]]
 === `_all` field
 
+deprecated::[6.0.0,The _all field is deprecated and will removed in Elasticsearch 6.0]
+
 The `_all` field is a special _catch-all_ field which concatenates the values
 of all of the other fields into one big string, using space as a delimiter, which is then
 <<analysis,analyzed>> and indexed, but not stored.  This means that it can be
@@ -119,6 +121,7 @@ PUT my_index
 --------------------------------
 // CONSOLE
 // TEST[s/\.\.\.//]
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 
 <1> The `_all` field in `type_1` is enabled.
 <2> The `_all` field in `type_2` is completely disabled.
@@ -150,6 +153,7 @@ PUT my_index
 }
 --------------------------------
 // CONSOLE
+// TEST[warning:The [_all] field is deprecated and will be removed in Elasticsearch 6.0]
 
 <1> The `_all` field is disabled for the `my_type` type.
 <2> The `query_string` query will default to querying the `content` field in this index.

rest-api-spec/src/main/resources/rest-api-spec/test/count/20_query_string.yaml

@@ -1,6 +1,12 @@
 ---
 "count with query_string parameters":
+  - skip:
+      features:
+        - warnings
+
   - do:
+      warnings:
+        - 'The [_all] field is deprecated and will be removed in Elasticsearch 6.0'
       indices.create:
           index:  test
           body:

rest-api-spec/src/main/resources/rest-api-spec/test/explain/30_query_string.yaml

@@ -1,6 +1,12 @@
 ---
 "explain with query_string parameters":
+  - skip:
+      features:
+        - warnings
+
   - do:
+      warnings:
+        - 'The [_all] field is deprecated and will be removed in Elasticsearch 6.0'
       indices.create:
           index:  test
           body:

rest-api-spec/src/main/resources/rest-api-spec/test/indices.validate_query/20_query_string.yaml

@@ -1,6 +1,12 @@
 ---
 "validate_query with query_string parameters":
+  - skip:
+      features:
+        - warnings
+
   - do:
+      warnings:
+        - 'The [_all] field is deprecated and will be removed in Elasticsearch 6.0'
       indices.create:
           index:  test
           body:

rest-api-spec/src/main/resources/rest-api-spec/test/search/60_query_string.yaml

@@ -1,6 +1,12 @@
 ---
 "search with query_string parameters":
+  - skip:
+      features:
+        - warnings
+
   - do:
+      warnings:
+        - 'The [_all] field is deprecated and will be removed in Elasticsearch 6.0'
       indices.create:
           index:  test
           body: