Merge remote-tracking branch 'es/master' into ccr

* es/master: (23 commits)
  Switch full-cluster-restart to new style Requests (#32140)
  [DOCS] Clarified that you must remove X-Pack plugin when upgrading from pre-6.3. (#32016)
  Remove BouncyCastle dependency from runtime (#32193)
  INGEST: Extend KV Processor (#31789) (#32232)
  INGEST: Make a few Processors callable by Painless (#32170)
  Add region ISO code to GeoIP Ingest plugin (#31669)
  [Tests] Remove QueryStringQueryBuilderTests#toQuery class assertions (#32236)
  Make sure that field aliases count towards the total fields limit. (#32222)
  Switch rolling restart to new style Requests (#32147)
  muting failing test for internal auto date histogram to avoid failure before fix is merged
  MINOR: Remove unused `IndexDynamicSettings` (#32237)
  Fix multi level nested sort (#32204)
  Enhance Parent circuit breaker error message (#32056)
  [ML] Use default request durability for .ml-state index (#32233)
  Remove indices stats timeout from monitoring docs
  Rename ranking evaluation response section (#32166)
  Dependencies: Upgrade to joda time 2.10 (#32160)
  Remove aliases resolution limitations when security is enabled (#31952)
  Ensure that field aliases cannot be used in multi-fields. (#32219)
  TESTS: Check for Netty resource leaks (#31861)
  ...

buildSrc/build.gradle

@@ -25,8 +25,8 @@ plugins {
 
 group = 'org.elasticsearch.gradle'
 
-if (GradleVersion.current() < GradleVersion.version('3.3')) {
-  throw new GradleException('Gradle 3.3+ is required to build elasticsearch')
+if (GradleVersion.current() < GradleVersion.version('4.9')) {
+  throw new GradleException('Gradle 4.9+ is required to build elasticsearch')
 }
 
 if (JavaVersion.current() < JavaVersion.VERSION_1_8) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -67,6 +67,9 @@ class BuildPlugin implements Plugin<Project> {
                 + 'elasticearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
+        if (GradleVersion.current() < GradleVersion.version('4.9')) {
+            throw new GradleException('Gradle 4.9+ is required to use elasticsearch.build plugin')
+        }
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
         // these plugins add lots of info to our jars

client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java

@@ -40,7 +40,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnratedDocuments;
 
 public class RankEvalIT extends ESRestHighLevelClientTestCase {
 
@@ -84,7 +84,7 @@ public void testRankEvalRequest() throws IOException {
         Map<String, EvalQueryQuality> partialResults = response.getPartialResults();
         assertEquals(2, partialResults.size());
         EvalQueryQuality amsterdamQueryQuality = partialResults.get("amsterdam_query");
-        assertEquals(2, filterUnknownDocuments(amsterdamQueryQuality.getHitsAndRatings()).size());
+        assertEquals(2, filterUnratedDocuments(amsterdamQueryQuality.getHitsAndRatings()).size());
         List<RatedSearchHit> hitsAndRatings = amsterdamQueryQuality.getHitsAndRatings();
         assertEquals(7, hitsAndRatings.size());
         for (RatedSearchHit hit : hitsAndRatings) {
@@ -96,7 +96,7 @@ public void testRankEvalRequest() throws IOException {
             }
         }
         EvalQueryQuality berlinQueryQuality = partialResults.get("berlin_query");
-        assertEquals(6, filterUnknownDocuments(berlinQueryQuality.getHitsAndRatings()).size());
+        assertEquals(6, filterUnratedDocuments(berlinQueryQuality.getHitsAndRatings()).size());
         hitsAndRatings = berlinQueryQuality.getHitsAndRatings();
         assertEquals(7, hitsAndRatings.size());
         for (RatedSearchHit hit : hitsAndRatings) {

distribution/archives/build.gradle

@@ -49,7 +49,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles(oss)
       }
       into('config') {
         dirMode 0750

distribution/build.gradle

@@ -227,16 +227,24 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles = copySpec {
-      // delay by using closures, since they have not yet been configured, so no jar task exists yet
-      from { project(':server').jar }
-      from { project(':server').configurations.runtime }
-      from { project(':libs:plugin-classloader').jar }
-      from { project(':distribution:tools:java-version-checker').jar }
-      from { project(':distribution:tools:launchers').jar }
-      into('tools/plugin-cli') {
-        from { project(':distribution:tools:plugin-cli').jar }
-        from { project(':distribution:tools:plugin-cli').configurations.runtime }
+    libFiles = { oss ->
+      copySpec {
+        // delay by using closures, since they have not yet been configured, so no jar task exists yet
+        from { project(':server').jar }
+        from { project(':server').configurations.runtime }
+        from { project(':libs:plugin-classloader').jar }
+        from { project(':distribution:tools:java-version-checker').jar }
+        from { project(':distribution:tools:launchers').jar }
+        into('tools/plugin-cli') {
+          from { project(':distribution:tools:plugin-cli').jar }
+          from { project(':distribution:tools:plugin-cli').configurations.runtime }
+        }
+        if (oss == false) {
+          into('tools/security-cli') {
+            from { project(':x-pack:plugin:security:cli').jar }
+            from { project(':x-pack:plugin:security:cli').configurations.compile }
+          }
+        }
       }
     }
 

distribution/packages/build.gradle

@@ -126,7 +126,7 @@ Closure commonPackageConfig(String type, boolean oss) {
       }
       into('lib') {
         with copySpec {
-          with libFiles
+          with libFiles(oss)
           // we need to specify every intermediate directory so we iterate through the parents; duplicate calls with the same part are fine
           eachFile { FileCopyDetails fcp ->
             String[] segments = fcp.relativePath.segments

docs/plugins/ingest-geoip.asciidoc

@@ -26,14 +26,14 @@ include::install_remove.asciidoc[]
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
 | `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb, GeoLite2-Country.mmdb and GeoLite2-ASN.mmdb files.
-| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
+| `properties`           | no        | [`continent_name`, `country_iso_code`, `region_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 | `ignore_missing`       | no        | `false`                                                                            | If `true` and `field` does not exist, the processor quietly exits without modifying the document
 |======
 
 *Depends on what is available in `database_field`:
 
 * If the GeoLite2 City database is used, then the following fields may be added under the `target_field`: `ip`,
-`country_iso_code`, `country_name`, `continent_name`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
+`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `latitude`, `longitude`
 and `location`. The fields actually added depend on what has been found and which properties were configured in `properties`.
 * If the GeoLite2 Country database is used, then the following fields may be added under the `target_field`: `ip`,
 `country_iso_code`, `country_name` and `continent_name`. The fields actually added depend on what has been found and which properties

docs/reference/ingest/ingest-node.asciidoc

@@ -1732,6 +1732,10 @@ For example, if you have a log message which contains `ip=1.2.3.4 error=REFUSED`
 | `include_keys`   | no        | `null`   | List of keys to filter and insert into document. Defaults to including all keys
 | `exclude_keys`   | no        | `null`   | List of keys to exclude from document
 | `ignore_missing` | no        | `false`  | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document
+| `prefix`         | no        | `null`   | Prefix to be added to extracted keys
+| `trim_key`       | no        | `null`   | String of characters to trim from extracted keys
+| `trim_value`     | no        | `null`   | String of characters to trim from extracted values
+| `strip_brackets` | no        | `false`  | If `true` strip brackets `()`, `<>`, `[]` as well as quotes `'` and `"` from extracted values
 |======
 
 

docs/reference/mapping/types/alias.asciidoc

@@ -74,7 +74,7 @@ field alias to query over multiple target fields in a single clause.
 ==== Unsupported APIs
 
 Writes to field aliases are not supported: attempting to use an alias in an index or update request
-will result in a failure. Likewise, aliases cannot be used as the target of `copy_to`.
+will result in a failure. Likewise, aliases cannot be used as the target of `copy_to` or in multi-fields.
 
 Because alias names are not present in the document source, aliases cannot be used when performing
 source filtering. For example, the following request will return an empty result for `_source`:

docs/reference/migration/migrate_7_0/api.asciidoc

@@ -79,3 +79,11 @@ the only behavior in 8.0.0, this parameter is deprecated in 7.0.0 for removal in
 ==== The deprecated stored script contexts have now been removed
 When putting stored scripts, support for storing them with the deprecated `template` context or without a context is
 now removed. Scripts must be stored using the `script` context as mentioned in the documentation.
+
+==== Get Aliases API limitations when {security} is enabled removed
+
+The behavior and response codes of the get aliases API no longer vary
+depending on whether {security} is enabled. Previously a
+404 - NOT FOUND (IndexNotFoundException) could be returned in case the
+current user was not authorized for any alias. An empty response with
+status 200 - OK is now returned instead at all times.

docs/reference/search/rank-eval.asciidoc

@@ -274,7 +274,7 @@ that shows potential errors of individual queries. The response has the followin
         "details": {  
             "my_query_id1": { <2>
                 "quality_level": 0.6, <3>
-                "unknown_docs": [ <4>
+                "unrated_docs": [ <4>
                     {
                         "_index": "my_index",
                         "_id": "1960795"
@@ -309,7 +309,7 @@ that shows potential errors of individual queries. The response has the followin
 <1> the overall evaluation quality calculated by the defined metric
 <2> the `details` section contains one entry for every query in the original `requests` section, keyed by the search request id
 <3> the `quality_level` in the `details` section shows the contribution of this query to the global quality score
-<4> the `unknown_docs` section contains an `_index` and `_id` entry for each document in the search result for this
+<4> the `unrated_docs` section contains an `_index` and `_id` entry for each document in the search result for this
 query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents
 <5> the `hits` section shows a grouping of the search results with their supplied rating
 <6> the `metric_details` give additional information about the calculated quality metric (e.g. how many of the retrieved

docs/reference/settings/monitoring-settings.asciidoc

@@ -85,10 +85,6 @@ You can update this setting through the
 
 Sets the timeout for collecting index statistics. Defaults to `10s`.
 
-`xpack.monitoring.collection.indices.stats.timeout`::
-
-Sets the timeout for collecting total indices statistics. Defaults to `10s`.
-
 `xpack.monitoring.collection.index.recovery.active_only`::
 
 Controls whether or not all recoveries are collected. Set to `true` to

docs/reference/upgrade/cluster_restart.asciidoc

@@ -47,6 +47,8 @@ include::set-paths-tip.asciidoc[]
 Use the `elasticsearch-plugin` script to install the upgraded version of each
 installed Elasticsearch plugin. All plugins must be upgraded when you upgrade
 a node.
++
+include::remove-xpack.asciidoc[]
 
 . *Start each upgraded node.*
 +
@@ -91,7 +93,7 @@ already have local shard copies.
 +
 --
 When all nodes have joined the cluster and recovered their primary shards,
-reenable allocation by restoring `cluster.routing.allocation.enable` to its 
+reenable allocation by restoring `cluster.routing.allocation.enable` to its
 default:
 
 [source,js]
@@ -123,4 +125,4 @@ GET _cat/recovery
 // CONSOLE
 --
 
-. *Restart machine learning jobs.* 
+. *Restart machine learning jobs.*

docs/reference/upgrade/remove-xpack.asciidoc

@@ -0,0 +1,4 @@
+IMPORTANT: If you use {xpack} and are upgrading from a version prior to 6.3,
+remove {xpack} before restarting: `bin/elasticsearch-plugin remove x-pack`. As
+of 6.3, {xpack} is included in the default distribution. The node will fail to
+start if the old {xpack} plugin is present.

docs/reference/upgrade/rolling_upgrade.asciidoc

@@ -53,6 +53,8 @@ include::set-paths-tip.asciidoc[]
 Use the `elasticsearch-plugin` script to install the upgraded version of each
 installed Elasticsearch plugin. All plugins must be upgraded when you upgrade
 a node.
++
+include::remove-xpack.asciidoc[]
 
 . *Start the upgraded node.*
 +
@@ -144,7 +146,7 @@ for each node that needs to be updated.
 
 --
 
-. *Restart machine learning jobs.* 
+. *Restart machine learning jobs.*
 
 [IMPORTANT]
 ====================================================

modules/ingest-common/build.gradle

@@ -20,11 +20,17 @@
 esplugin {
     description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
     classname 'org.elasticsearch.ingest.common.IngestCommonPlugin'
+    extendedPlugins = ['lang-painless']
 }
 
 dependencies {
+    compileOnly project(':modules:lang-painless')
     compile project(':libs:grok')
 }
 
 compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
 compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
+
+integTestCluster {
+    module project(':modules:lang-painless')
+}

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/BytesProcessor.java

@@ -35,9 +35,13 @@ public final class BytesProcessor extends AbstractStringProcessor {
         super(processorTag, field, ignoreMissing, targetField);
     }
 
+    public static long apply(String value) {
+        return ByteSizeValue.parseBytesSizeValue(value, null, "Ingest Field").getBytes();
+    }
+
     @Override
     protected Long process(String value) {
-        return ByteSizeValue.parseBytesSizeValue(value, null, getField()).getBytes();
+        return apply(value);
     }
 
     @Override

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java

@@ -67,13 +67,11 @@ boolean isAddToRoot() {
         return addToRoot;
     }
 
-    @Override
-    public void execute(IngestDocument document) throws Exception {
-        Object fieldValue = document.getFieldValue(field, Object.class);
-        BytesReference bytesRef = (fieldValue == null) ? new BytesArray("null") : new BytesArray(fieldValue.toString());
+    public static Object apply(Object fieldValue) {
+        BytesReference bytesRef = fieldValue == null ? new BytesArray("null") : new BytesArray(fieldValue.toString());
         try (InputStream stream = bytesRef.streamInput();
              XContentParser parser = JsonXContent.jsonXContent
-                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
+                 .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, stream)) {
             XContentParser.Token token = parser.nextToken();
             Object value = null;
             if (token == XContentParser.Token.VALUE_NULL) {
@@ -91,20 +89,32 @@ public void execute(IngestDocument document) throws Exception {
             } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                 throw new IllegalArgumentException("cannot read binary value");
             }
-            if (addToRoot && (value instanceof Map)) {
-                for (Map.Entry<String, Object> entry : ((Map<String, Object>) value).entrySet()) {
-                    document.setFieldValue(entry.getKey(), entry.getValue());
-                }
-            } else if (addToRoot) {
-                throw new IllegalArgumentException("cannot add non-map fields to root of document");
-            } else {
-                document.setFieldValue(targetField, value);
-            }
+            return value;
         } catch (IOException e) {
             throw new IllegalArgumentException(e);
         }
     }
 
+    public static void apply(Map<String, Object> ctx, String fieldName) {
+        Object value = apply(ctx.get(fieldName));
+        if (value instanceof Map) {
+            @SuppressWarnings("unchecked")
+                Map<String, Object> map = (Map<String, Object>) value;
+                ctx.putAll(map);
+        } else {
+            throw new IllegalArgumentException("cannot add non-map fields to root of document");
+        }
+    }
+
+    @Override
+    public void execute(IngestDocument document) throws Exception {
+        if (addToRoot) {
+           apply(document.getSourceAndMetadata(), field);
+        } else {
+            document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class)));
+        }
+    }
+
     @Override
     public String getType() {
         return TYPE;