Merge branch 'master' into ccr

* master: (24 commits)
  Watcher: Ensure mail message ids are unique per watch action (#30112)
  REST: Remove GET support for clear cache indices (#29525)
  SQL: Correct error message (#30138)
  Require acknowledgement to start_trial license (#30135)
  Fix a bug in FieldCapabilitiesRequest#equals and hashCode. (#30181)
  SQL: Add BinaryMathProcessor to named writeables list (#30127)
  Tests: Use buildDir as base for generated-resources (#30191)
  Fix SliceBuilderTests#testRandom failures
  Build: Fix deb version to use tilde with prerelease versions (#29000)
  Fix edge cases in CompositeKeyExtractorTests (#30175)
  Document time unit limitations for date histograms (#30177)
  Add support for field capabilities to the high-level REST client. (#29664)
  Remove licenses missed by the migration (#30128)
  [DOCS] Updates docker installation package details (#30110)
  Fix TermsSetQueryBuilder.doEquals() method (#29629)
  [Monitoring] Remove unhelpful Monitoring tests (#30144)
  [Test] Fix RenameProcessorTests.testRenameExistingFieldNullValue() (#29655)
  add copyright/scope configuration for intellij to Contributing Guide (#29688)
  [test] include oss tar in packaging tests (#30155)
  TEST: Update settings should go through cluster state (#29682)
  ...

CONTRIBUTING.md

@@ -107,6 +107,8 @@ We support development in the Eclipse and IntelliJ IDEs. For Eclipse, the
 minimum version that we support is [Eclipse Oxygen][eclipse] (version 4.7). For
 IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
 
+### Configuring IDEs And Running Tests
+
 Eclipse users can automatically configure their IDE: `./gradlew eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the
 option `Search for nested projects`. Additionally you will want to
@@ -144,6 +146,9 @@ For IntelliJ, go to
 For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 `VM Arguments`.
 
+
+### Java Language Formatting Guidelines
+
 Please follow these formatting guidelines:
 
 * Java indent is 4 spaces
@@ -155,6 +160,33 @@ Please follow these formatting guidelines:
  * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
 * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
 
+### License Headers
+
+We require license headers on all Java files. You will notice that all the Java files in
+the top-level `x-pack` directory contain a separate license from the rest of the repository. This
+directory contains commercial code that is associated with a separate license. It can be helpful
+to have the IDE automatically insert the appropriate license header depending which part of the project
+contributions are made to.
+
+#### IntelliJ: Copyright & Scope Profiles
+
+To have IntelliJ insert the correct license, it is necessary to create to copyright profiles.
+These may potentially be called `apache2` and `commercial`. These can be created in
+`Preferences/Settings->Editor->Copyright->Copyright Profiles`. To associate these profiles to
+their respective directories, two "Scopes" will need to be created. These can be created in
+`Preferences/Settings->Appearances & Behavior->Scopes`. When creating scopes, be sure to choose
+the `shared` scope type. Create a scope, `apache2`, with
+the associated pattern of `!file[group:x-pack]:*/`. This pattern will exclude all the files contained in
+the `x-pack` directory. The other scope, `commercial`, will have the inverse pattern of `file[group:x-pack]:*/`.
+The two scopes, together, should account for all the files in the project. To associate the scopes
+with their copyright-profiles, go into `Preferences/Settings->Editor>Copyright` and use the `+` to add
+the associations `apache2/apache2` and `commercial/commercial`.
+
+Configuring these options in IntelliJ can be quite buggy, so do not be alarmed if you have to open/close
+the settings window and/or restart IntelliJ to see your changes take effect.
+
+### Creating A Distribution
+
 To create a distribution from the source, simply run:
 
 ```sh
@@ -169,6 +201,8 @@ The archive distributions (tar and zip) can be found under:
 `./distribution/archives/(tar|zip)/build/distributions/`
 
 
+### Running The Full Test Suite
+
 Before submitting your changes, run the test suite to make sure that nothing is broken, with:
 
 ```sh

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -37,8 +37,15 @@ class VagrantTestPlugin implements Plugin<Project> {
             'ubuntu-1404',
     ]
 
-    /** All onboarded archives by default, available for Bats tests even if not used **/
-    static List<String> DISTRIBUTION_ARCHIVES = ['tar', 'rpm', 'deb', 'oss-rpm', 'oss-deb']
+    /** All distributions to bring into test VM, whether or not they are used **/
+    static List<String> DISTRIBUTIONS = [
+            'archives:tar',
+            'archives:oss-tar',
+            'packages:rpm',
+            'packages:oss-rpm',
+            'packages:deb',
+            'packages:oss-deb'
+    ]
 
     /** Packages onboarded for upgrade tests **/
     static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
@@ -117,13 +124,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
-        DISTRIBUTION_ARCHIVES.each {
+        DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            if (it == 'tar') {
-                it = 'archives:tar'
-            } else {
-                it = "packages:${it}"
-            }
             project.dependencies.add(PACKAGING_CONFIGURATION,
                     project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -535,7 +535,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReplicaShardAllocatorTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReusePeerRecoverySharedTest.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]get[/\\]GetActionIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexServiceTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicySettingsTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLogTests.java" checks="LineLength" />

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -48,6 +48,7 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -75,6 +76,7 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.rest.action.RestFieldCapabilitiesAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
@@ -536,6 +538,16 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
         return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null);
     }
 
+    static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) {
+        Params params = Params.builder();
+        params.withFields(fieldCapabilitiesRequest.fields());
+        params.withIndicesOptions(fieldCapabilitiesRequest.indicesOptions());
+
+        String[] indices = fieldCapabilitiesRequest.indices();
+        String endpoint = endpoint(indices, "_field_caps");
+        return new Request(HttpGet.METHOD_NAME, endpoint, params.getParams(), null);
+    }
+
     static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
         String endpoint = endpoint(rankEvalRequest.indices(), Strings.EMPTY_ARRAY, "_rank_eval");
         Params params = Params.builder();
@@ -712,6 +724,13 @@ Params withFetchSourceContext(FetchSourceContext fetchSourceContext) {
             return this;
         }
 
+        Params withFields(String[] fields) {
+            if (fields != null && fields.length > 0) {
+                return putParam("fields", String.join(",", fields));
+            }
+            return this;
+        }
+
         Params withMasterTimeout(TimeValue masterTimeout) {
             return putParam("master_timeout", masterTimeout);
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -30,6 +30,8 @@
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
@@ -501,6 +503,31 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener<
                 headers);
     }
 
+    /**
+     * Executes a request using the Field Capabilities API.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html">Field Capabilities API
+     * on elastic.co</a>.
+     */
+    public final FieldCapabilitiesResponse fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest,
+                                                     Header... headers) throws IOException {
+        return performRequestAndParseEntity(fieldCapabilitiesRequest, Request::fieldCaps,
+            FieldCapabilitiesResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a request using the Field Capabilities API.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html">Field Capabilities API
+     * on elastic.co</a>.
+     */
+    public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesRequest,
+                                     ActionListener<FieldCapabilitiesResponse> listener,
+                                     Header... headers) {
+        performRequestAsyncAndParseEntity(fieldCapabilitiesRequest, Request::fieldCaps,
+            FieldCapabilitiesResponse::fromXContent, listener, emptySet(), headers);
+    }
+
     protected final <Req extends ActionRequest, Resp> Resp performRequestAndParseEntity(Req request,
                                                                             CheckedFunction<Req, Request, IOException> requestConverter,
                                                                             CheckedFunction<XContentParser, Resp, IOException> entityParser,

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -52,6 +52,7 @@
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkShardRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
@@ -89,6 +90,7 @@
 import org.elasticsearch.index.rankeval.RankEvalSpec;
 import org.elasticsearch.index.rankeval.RatedRequest;
 import org.elasticsearch.index.rankeval.RestRankEvalAction;
+import org.elasticsearch.rest.action.RestFieldCapabilitiesAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
@@ -108,11 +110,14 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.StringJoiner;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -128,6 +133,8 @@
 import static org.elasticsearch.search.RandomSearchRequestGenerator.randomSearchRequest;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.nullValue;
 
 public class RequestTests extends ESTestCase {
@@ -1213,6 +1220,47 @@ public void testExistsAliasNoAliasNoIndex() {
         }
     }
 
+    public void testFieldCaps() {
+        // Create a random request.
+        String[] indices = randomIndicesNames(0, 5);
+        String[] fields = generateRandomStringArray(5, 10, false, false);
+
+        FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest()
+            .indices(indices)
+            .fields(fields);
+
+        Map<String, String> indicesOptionsParams = new HashMap<>();
+        setRandomIndicesOptions(fieldCapabilitiesRequest::indicesOptions,
+            fieldCapabilitiesRequest::indicesOptions,
+            indicesOptionsParams);
+
+        Request request = Request.fieldCaps(fieldCapabilitiesRequest);
+
+        // Verify that the resulting REST request looks as expected.
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String joinedIndices = String.join(",", indices);
+        if (!joinedIndices.isEmpty()) {
+            endpoint.add(joinedIndices);
+        }
+        endpoint.add("_field_caps");
+
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(4, request.getParameters().size());
+
+        // Note that we don't check the field param value explicitly, as field names are passed through
+        // a hash set before being added to the request, and can appear in a non-deterministic order.
+        assertThat(request.getParameters(), hasKey("fields"));
+        String[] requestFields = Strings.splitStringByCommaToArray(request.getParameters().get("fields"));
+        assertEquals(new HashSet<>(Arrays.asList(fields)),
+            new HashSet<>(Arrays.asList(requestFields)));
+
+        for (Map.Entry<String, String> param : indicesOptionsParams.entrySet()) {
+            assertThat(request.getParameters(), hasEntry(param.getKey(), param.getValue()));
+        }
+
+        assertNull(request.getEntity());
+    }
+
     public void testRankEval() throws Exception {
         RankEvalSpec spec = new RankEvalSpec(
                 Collections.singletonList(new RatedRequest("queryId", Collections.emptyList(), new SearchSourceBuilder())),
@@ -1233,7 +1281,6 @@ public void testRankEval() throws Exception {
         assertEquals(3, request.getParameters().size());
         assertEquals(expectedParams, request.getParameters());
         assertToXContentBody(spec, request.getEntity());
-
     }
 
     public void testSplit() throws IOException {

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -27,6 +27,9 @@
 import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
 import org.elasticsearch.action.search.MultiSearchRequest;
@@ -96,14 +99,31 @@ public void indexDocuments() throws IOException {
         client().performRequest(HttpPut.METHOD_NAME, "/index/type/5", Collections.emptyMap(), doc5);
         client().performRequest(HttpPost.METHOD_NAME, "/index/_refresh");
 
-        StringEntity doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
+
+        StringEntity doc = new StringEntity("{\"field\":\"value1\", \"rating\": 7}", ContentType.APPLICATION_JSON);
         client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/1", Collections.emptyMap(), doc);
         doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
         client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/2", Collections.emptyMap(), doc);
-        doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
+
+        StringEntity mappings = new StringEntity(
+            "{" +
+            "  \"mappings\": {" +
+            "    \"doc\": {" +
+            "      \"properties\": {" +
+            "        \"rating\": {" +
+            "          \"type\":  \"keyword\"" +
+            "        }" +
+            "      }" +
+            "    }" +
+            "  }" +
+            "}}",
+            ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index2", Collections.emptyMap(), mappings);
+        doc = new StringEntity("{\"field\":\"value1\", \"rating\": \"good\"}", ContentType.APPLICATION_JSON);
         client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/3", Collections.emptyMap(), doc);
         doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
         client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/4", Collections.emptyMap(), doc);
+
         doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON);
         client().performRequest(HttpPut.METHOD_NAME, "/index3/doc/5", Collections.emptyMap(), doc);
         doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON);
@@ -713,6 +733,57 @@ public void testMultiSearch_failure() throws Exception {
         assertThat(multiSearchResponse.getResponses()[1].getResponse(), nullValue());
     }
 
+    public void testFieldCaps() throws IOException {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
+            .indices("index1", "index2")
+            .fields("rating", "field");
+
+        FieldCapabilitiesResponse response = execute(request,
+            highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync);
+
+        // Check the capabilities for the 'rating' field.
+        assertTrue(response.get().containsKey("rating"));
+        Map<String, FieldCapabilities> ratingResponse = response.getField("rating");
+        assertEquals(2, ratingResponse.size());
+
+        FieldCapabilities expectedKeywordCapabilities = new FieldCapabilities(
+            "rating", "keyword", true, true, new String[]{"index2"}, null, null);
+        assertEquals(expectedKeywordCapabilities, ratingResponse.get("keyword"));
+
+        FieldCapabilities expectedLongCapabilities = new FieldCapabilities(
+            "rating", "long", true, true, new String[]{"index1"}, null, null);
+        assertEquals(expectedLongCapabilities, ratingResponse.get("long"));
+
+        // Check the capabilities for the 'field' field.
+        assertTrue(response.get().containsKey("field"));
+        Map<String, FieldCapabilities> fieldResponse = response.getField("field");
+        assertEquals(1, fieldResponse.size());
+
+        FieldCapabilities expectedTextCapabilities = new FieldCapabilities(
+            "field", "text", true, false);
+        assertEquals(expectedTextCapabilities, fieldResponse.get("text"));
+    }
+
+    public void testFieldCapsWithNonExistentFields() throws IOException {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
+            .indices("index2")
+            .fields("nonexistent");
+
+        FieldCapabilitiesResponse response = execute(request,
+            highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync);
+        assertTrue(response.get().isEmpty());
+    }
+
+    public void testFieldCapsWithNonExistentIndices() {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
+            .indices("non-existent")
+            .fields("rating");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> execute(request, highLevelClient()::fieldCaps, highLevelClient()::fieldCapsAsync));
+        assertEquals(RestStatus.NOT_FOUND, exception.status());
+    }
+
     private static void assertSearchHeader(SearchResponse searchResponse) {
         assertThat(searchResponse.getTook().nanos(), greaterThanOrEqualTo(0L));
         assertEquals(0, searchResponse.getFailedShards());