Add an include_type_name option. (elastic#29453)

This adds an `include_type_name` option to the `indices.create`,
`indices.get_mapping` and `indices.put_mapping` APIs, which defaults to `true`.
When set to `false`, then mappings will be returned directly in the body of
the `indices.get_mapping` API, without keying them by the type name, the
`indices.create` will expect mappings directly under the `mappings` key, and
the `indices.put_mapping` will use `_doc` as a type name and fail if a `type`
is provided explicitly.

On 5.x indices, get-mapping will fail if the index has multiple mappings, and
put-mapping will update or introduce mappings for the `_doc` type instead of
updating existing mappings. This oddity is required so that we don't have to
introduce a new flag to put-mapping requests to know whether they are actually
updating the `_doc` type or performing a typeless call.

Relates elastic#15613

docs/reference/indices/create-index.asciidoc

@@ -177,3 +177,29 @@ PUT test?wait_for_active_shards=2
 
 A detailed explanation of `wait_for_active_shards` and its possible values can be found
 <<index-wait-for-active-shards,here>>.
+
+[float]
+=== Skipping types
+
+Types are being removed from Elasticsearch: in 7.0, the `mappings` element will
+no longer take the type name as a top-level key. You can already opt in for this
+behavior by setting setting `include_type_name=false` and putting mappings
+directly under `mappings` in the index creation call, without specifying a type
+name.
+
+Here is an example:
+
+[source,js]
+--------------------------------------------------
+PUT test?include_type_name=false
+{
+  "mappings": {
+    "properties": {
+      "foo": {
+        "type": "keyword"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE

docs/reference/indices/get-mapping.asciidoc

@@ -41,3 +41,51 @@ GET /_mapping
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
+
+[float]
+=== Skipping types
+
+Types are being removed from Elasticsearch: in 7.0, the `mappings` element will
+no longer take the type name as a top-level key. You can already opt in for this
+behavior by setting setting `include_type_name=false` on the request.
+
+NOTE: Such calls will be rejected on indices that have multiple types as it
+introduces ambiguity as to which mapping should be returned. Only indices
+created by Elasticsearch 5.x may have multiple types.
+
+Here is an example:
+
+[source,js]
+--------------------------------------------------
+PUT test?include_type_name=false
+{
+  "mappings": {
+    "properties": {
+      "foo": {
+        "type": "keyword"
+      }
+    }
+  }
+}
+
+GET test/_mappings?include_type_name=false
+--------------------------------------------------
+// CONSOLE
+
+which returns
+
+[source,js]
+--------------------------------------------------
+{
+  "test": {
+    "mappings": {
+      "properties": {
+        "foo": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE

docs/reference/indices/put-mapping.asciidoc

@@ -114,3 +114,40 @@ PUT my_index/_mapping/_doc
 
 Each <<mapping-params,mapping parameter>> specifies whether or not its setting
 can be updated on an existing field.
+
+[float]
+=== Skipping types
+
+Types are being removed from Elasticsearch: in 7.0, the `mappings` element will
+no longer take the type name as a top-level key. You can already opt in for this
+behavior by setting setting `include_type_name=false` and putting mappings
+directly under `mappings` in the put mapping call, without specifying a type
+name.
+
+NOTE: On indices created on Elasticsearch 5.x, such calls will actually
+introduce or update mappings for the `_doc` type. It is recommended to avoid
+calling the put-mapping API with `include_type_name=false` on 5.x indices.
+
+Here is an example:
+
+[source,js]
+-----------------------------------
+PUT my_index?include_type_name=false <1>
+{
+  "mappings": {
+    "properties": {
+      "name": {
+        "properties": {
+          "first": {
+            "type": "text"
+          }
+        }
+      },
+      "user_id": {
+        "type": "keyword"
+      }
+    }
+  }
+}
+-----------------------------------
+// CONSOLE

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.hamcrest.Matchers;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -1003,6 +1004,115 @@ public void testSoftDeletes() throws Exception {
         }
     }
 
+    public void testIncludeTypeNameOnMultiTypesIndices() throws IOException {
+        assumeTrue("This test only makes sense if the old cluster runs a version that supports types",
+                getOldClusterVersion().before(Version.V_6_0_0));
+        if (isRunningAgainstOldCluster()) {
+            String mapping = Strings.toString(JsonXContent.contentBuilder()
+                    .startObject()
+                        .startObject("mappings")
+                            .startObject("type1")
+                                .startObject("properties")
+                                    .startObject("foo")
+                                        .field("type", "keyword")
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                            .startObject("type2")
+                                .startObject("properties")
+                                    .startObject("foo")
+                                        .field("type", "keyword")
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject());
+            Request createIndexRequest = new Request("PUT", "/" + index);
+            createIndexRequest.setJsonEntity(mapping);
+            client().performRequest(createIndexRequest);
+        } else {
+            // GET mappings
+            Request getMappingsRequest = new Request("GET", index + "/_mappings");
+            getMappingsRequest.addParameter("include_type_name", "false");
+            ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(getMappingsRequest));
+            assertThat(EntityUtils.toString(ex.getResponse().getEntity()), Matchers.containsString("has multiple mappings"));
+
+            // PUT mappings
+            Request putMappingsRequest = new Request("PUT", index + "/_mappings");
+            putMappingsRequest.addParameter("include_type_name", "false");
+            String mapping = Strings.toString(JsonXContent.contentBuilder()
+                    .startObject()
+                        .startObject("properties")
+                            .startObject("bar")
+                                .field("type", "long")
+                            .endObject()
+                        .endObject()
+                    .endObject());
+            putMappingsRequest.setJsonEntity(mapping);
+            client().performRequest(putMappingsRequest);
+
+            Request getMappingsRequest2 = new Request("GET", index + "/_mappings");
+            Response getMappingsResponse = client().performRequest(getMappingsRequest2);
+            String getMappingsResponseAsString = EntityUtils.toString(getMappingsResponse.getEntity());
+            // we introduced a _doc mapping
+            assertThat(getMappingsResponseAsString, Matchers.containsString("\"_doc\":{"));
+        }
+    }
+
+    public void testIncludeTypeNameOnSingleTypeIndices() throws IOException {
+        if (isRunningAgainstOldCluster()) {
+            String mapping = Strings.toString(JsonXContent.contentBuilder()
+                    .startObject()
+                        .startObject("mappings")
+                            .startObject("my_type")
+                                .startObject("properties")
+                                    .startObject("foo")
+                                        .field("type", "keyword")
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject());
+            Request request = new Request("PUT", "/" + index);
+            request.setJsonEntity(mapping);
+            client().performRequest(request);
+        } else {
+            // PUT mappings
+            Request putMappingsRequest = new Request("PUT", index + "/_mappings");
+            putMappingsRequest.addParameter("include_type_name", "false");
+            String mapping = Strings.toString(JsonXContent.contentBuilder()
+                    .startObject()
+                        .startObject("properties")
+                            .startObject("bar")
+                                .field("type", "long")
+                            .endObject()
+                        .endObject()
+                    .endObject());
+            putMappingsRequest.setJsonEntity(mapping);
+            client().performRequest(putMappingsRequest);
+
+            // GET mappings
+            Request getMappingsRequest = new Request("GET", index + "/_mappings");
+            getMappingsRequest.addParameter("include_type_name", "false");
+            if (getOldClusterVersion().before(Version.V_6_0_0)) {
+                ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(getMappingsRequest));
+                assertThat(EntityUtils.toString(ex.getResponse().getEntity()), Matchers.containsString("has multiple mappings"));
+                Request getMappingsRequest2 = new Request("GET", index + "/_mappings");
+                Response getMappingsResponse = client().performRequest(getMappingsRequest2);
+                String getMappingsResponseAsString = EntityUtils.toString(getMappingsResponse.getEntity());
+                // we introduced a _doc mapping
+                assertThat(getMappingsResponseAsString, Matchers.containsString("\"_doc\":{"));
+                assertThat(getMappingsResponseAsString, Matchers.containsString("\"my_type\":{"));
+            } else {
+                Response getMappingsResponse = client().performRequest(getMappingsRequest);
+                String getMappingsResponseAsString = EntityUtils.toString(getMappingsResponse.getEntity());
+                assertThat(getMappingsResponseAsString, Matchers.not(Matchers.containsString("my_type"))); // no type name
+                assertThat(getMappingsResponseAsString, Matchers.containsString("\"bar\":{\"type\":\"long\"}")); // new field is here
+                assertThat(getMappingsResponseAsString, Matchers.containsString("\"foo\":{\"type\":\"keyword\"}")); // old field is here too
+            }
+        }
+    }
+
     private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException {
         // Check the snapshot metadata, especially the version
         Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);

rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json

@@ -13,6 +13,10 @@
         }
       },
       "params": {
+        "include_type_name": {
+          "type" : "string",
+          "description" : "Whether a type should be expected in the body of the mappings."
+        },
         "wait_for_active_shards": {
           "type" : "string",
           "description" : "Set the number of active shards to wait for before the operation returns." 

rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json

@@ -16,6 +16,10 @@
         }
       },
       "params": {
+        "include_type_name": {
+          "type" : "string",
+          "description" : "Whether to add the type name to the response"
+        },
         "ignore_unavailable": {
             "type" : "boolean",
             "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"

rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json

@@ -4,19 +4,22 @@
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/{index}/{type}/_mapping",
-      "paths": ["/{index}/{type}/_mapping", "/{index}/_mapping/{type}", "/_mapping/{type}", "/{index}/{type}/_mappings", "/{index}/_mappings/{type}", "/_mappings/{type}"],
+      "paths": ["/{index}/{type}/_mapping", "/{index}/_mapping/{type}", "/_mapping/{type}", "/{index}/{type}/_mappings", "/{index}/_mappings/{type}", "/_mappings/{type}", "{index}/_mappings", "{index}/_mapping"],
       "parts": {
         "index": {
           "type" : "list",
           "description" : "A comma-separated list of index names the mapping should be added to (supports wildcards); use `_all` or omit to add the mapping on all indices."
         },
         "type": {
           "type" : "string",
-          "required" : true,
           "description" : "The name of the document type"
         }
       },
       "params": {
+        "include_type_name": {
+          "type" : "string",
+          "description" : "Whether a type should be expected in the body of the mappings."
+        },
         "timeout": {
           "type" : "time",
           "description" : "Explicit operation timeout"

rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_mapping/70_mix_typeless_typeful.yml

@@ -0,0 +1,23 @@
+---
+"GET mapping with typeless API on an index that has types":
+
+ - skip:
+      version: " - 6.6.99"
+      reason:  include_type_name was introduced in 6.7.0
+
+ - do:
+      indices.create: # not using include_type_name: false on purpose
+          index: index
+          body:
+              mappings:
+                  not_doc:
+                      properties:
+                          foo:
+                              type: "keyword"
+
+ - do:
+      indices.get_mapping:
+          include_type_name: false
+          index: index
+
+ - match: { index.mappings.properties.foo.type: "keyword" }

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml

@@ -0,0 +1,52 @@
+---
+"PUT mapping with typeless API on an index that has types":
+
+ - skip:
+      version: " - 6.6.99"
+      reason:  include_type_name was introduced in 6.7.0
+
+ - do:
+      indices.create: # not using include_type_name: false on purpose
+          index: index
+          body:
+              mappings:
+                  not_doc:
+                      properties:
+                          foo:
+                              type: "keyword"
+
+ - do:
+      indices.put_mapping:
+          include_type_name: false
+          index: index
+          body:
+              properties:
+                  bar:
+                      type: "long"
+
+ - do:
+      indices.get_mapping:
+          include_type_name: false
+          index: index
+
+ - match: { index.mappings.properties.foo.type: "keyword" }
+ - match: { index.mappings.properties.bar.type: "long" }
+
+ - do:
+      indices.put_mapping:
+          include_type_name: false
+          index: index
+          body:
+              properties:
+                  foo:
+                      type: "keyword" # also test no-op updates that trigger special logic wrt the mapping version
+
+ - do:
+      catch: bad_request
+      indices.put_mapping:
+          index: index
+          body:
+              some_other_type:
+                  properties:
+                      bar:
+                          type: "long"