Merge branch '6.x' into ccr-6.x

* 6.x:
  test: Break apart the multi type aspect of rolling upgrade tests,
  Upgrade to Jackson 2.8.10 (#27230)
  [Docs] Fix minor paragraph indentation error for multiple Indices params (#25535)
  Fix inconsistencies in the rest api specs for `tasks` (#27163)
  Adjust RestHighLevelClient method modifiers (#27238)
  Add more information on `_failed_to_convert_` exception (#27034)
  Remove ElasticsearchQueryCachingPolicy (#27190)
  Backport the size-based index rollver to v6.1.0
  Add size-based condition to the index rollover API (#27160)
  Remove the single argument Environment constructor (#27235)
  Fix RestGetAction name typo

buildSrc/version.properties

@@ -5,8 +5,8 @@ lucene            = 7.1.0
 # optional dependencies
 spatial4j         = 0.6
 jts               = 1.13
-jackson           = 2.8.6
-snakeyaml         = 1.15
+jackson           = 2.8.10
+snakeyaml         = 1.17
 # when updating log4j, please update also docs/java-api/index.asciidoc
 log4j             = 2.9.1
 slf4j             = 1.6.2

client/sniffer/licenses/jackson-core-2.8.10.jar.sha1

@@ -0,0 +1 @@
+eb21a035c66ad307e66ec8fce37f5d50fd62d039

core/licenses/jackson-core-2.8.10.jar.sha1

@@ -0,0 +1 @@
+eb21a035c66ad307e66ec8fce37f5d50fd62d039

core/licenses/jackson-dataformat-cbor-2.8.10.jar.sha1

@@ -0,0 +1 @@
+1c58cc9313ddf19f0900cd61ed044874278ce320

core/licenses/jackson-dataformat-smile-2.8.10.jar.sha1

@@ -0,0 +1 @@
+e853081fadaad3e98ed801937acc3d8f77580686

core/licenses/jackson-dataformat-yaml-2.8.10.jar.sha1

@@ -0,0 +1 @@
+1e08caf1d787c825307d8cc6362452086020d853

core/licenses/snakeyaml-1.17.jar.sha1

@@ -0,0 +1 @@
+7a27ea250c5130b2922b86dea63cbb1cc10a660c

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java

@@ -19,8 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -38,6 +40,9 @@ public abstract class Condition<T> implements NamedWriteable {
             new ParseField(MaxAgeCondition.NAME));
         PARSER.declareLong((conditions, value) ->
             conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
+        PARSER.declareString((conditions, s) ->
+                conditions.add(new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))),
+            new ParseField(MaxSizeCondition.NAME));
     }
 
     protected T value;
@@ -49,6 +54,14 @@ protected Condition(String name) {
 
     public abstract Result evaluate(Stats stats);
 
+    /**
+     * Checks if this condition is available in a specific version.
+     * This makes sure BWC when introducing a new condition which is not recognized by older versions.
+     */
+    boolean includedInVersion(Version version) {
+        return true;
+    }
+
     @Override
     public final String toString() {
         return "[" + name + ": " + value + "]";
@@ -60,10 +73,12 @@ public final String toString() {
     public static class Stats {
         public final long numDocs;
         public final long indexCreated;
+        public final ByteSizeValue indexSize;
 
-        public Stats(long numDocs, long indexCreated) {
+        public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
             this.numDocs = numDocs;
             this.indexCreated = indexCreated;
+            this.indexSize = indexSize;
         }
     }
 

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxSizeCondition.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+
+import java.io.IOException;
+
+/**
+ * A size-based condition for an index size.
+ * Evaluates to <code>true</code> if the index size is at least {@link #value}.
+ */
+public class MaxSizeCondition extends Condition<ByteSizeValue> {
+    public static final String NAME = "max_size";
+
+    public MaxSizeCondition(ByteSizeValue value) {
+        super(NAME);
+        this.value = value;
+    }
+
+    public MaxSizeCondition(StreamInput in) throws IOException {
+        super(NAME);
+        this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES);
+    }
+
+    @Override
+    public Result evaluate(Stats stats) {
+        return new Result(this, stats.indexSize.getBytes() >= value.getBytes());
+    }
+
+    @Override
+    boolean includedInVersion(Version version) {
+        return version.onOrAfter(Version.V_6_1_0);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(value.getBytes());
+    }
+}

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
@@ -106,7 +107,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(dryRun);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
-            out.writeNamedWriteable(condition);
+            if (condition.includedInVersion(out.getVersion())) {
+                out.writeNamedWriteable(condition);
+            }
         }
         createIndexRequest.writeTo(out);
     }
@@ -155,6 +158,13 @@ public void addMaxIndexDocsCondition(long numDocs) {
         this.conditions.add(new MaxDocsCondition(numDocs));
     }
 
+    /**
+     * Adds a size-based condition to check if the index size is at least <code>size</code>.
+     */
+    public void addMaxIndexSizeCondition(ByteSizeValue size) {
+        this.conditions.add(new MaxSizeCondition(size));
+    }
+
     /**
      * Sets rollover index creation request to override index settings when
      * the rolled over index has to be created

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
 
@@ -52,6 +53,11 @@ public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) {
         return this;
     }
 
+    public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){
+        this.request.addMaxIndexSizeCondition(size);
+        return this;
+    }
+
     public RolloverRequestBuilder dryRun(boolean dryRun) {
         this.request.dryRun(dryRun);
         return this;

core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java

@@ -43,6 +43,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -195,7 +196,8 @@ static String generateRolloverIndexName(String sourceIndexName, IndexNameExpress
     static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
                                                     final DocsStats docsStats, final IndexMetaData metaData) {
         final long numDocs = docsStats == null ? 0 : docsStats.getCount();
-        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate());
+        final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
+        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
         return conditions.stream()
             .map(condition -> condition.evaluate(stats))
             .collect(Collectors.toSet());

core/src/main/java/org/elasticsearch/common/io/FastStringReader.java

@@ -34,6 +34,7 @@ public class FastStringReader extends Reader implements CharSequence {
     private int length;
     private int next = 0;
     private int mark = 0;
+    private boolean closed = false;
 
     /**
      * Creates a new string reader.
@@ -49,8 +50,9 @@ public FastStringReader(String s) {
      * Check to make sure that the stream has not been closed
      */
     private void ensureOpen() throws IOException {
-        if (length == -1)
+        if (closed) {
             throw new IOException("Stream closed");
+        }
     }
 
     @Override
@@ -196,7 +198,7 @@ public void reset() throws IOException {
      */
     @Override
     public void close() {
-        length = -1;
+        closed = true;
     }
 
     @Override

core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -145,7 +145,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexModule.INDEX_STORE_PRE_LOAD_SETTING,
         IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING,
         IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING,
-        IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING,
         FsDirectoryService.INDEX_LOCK_FACTOR_SETTING,
         EngineConfig.INDEX_CODEC_SETTING,
         EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,

core/src/main/java/org/elasticsearch/env/Environment.java

@@ -85,10 +85,6 @@ public class Environment {
     /** Path to the temporary file directory used by the JDK */
     private final Path tmpFile = PathUtils.get(System.getProperty("java.io.tmpdir"));
 
-    public Environment(Settings settings) {
-        this(settings, null);
-    }
-
     public Environment(final Settings settings, final Path configPath) {
         final Path homeFile;
         if (PATH_HOME_SETTING.exists(settings)) {

core/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -101,11 +101,6 @@ public final class IndexModule {
     public static final Setting<Boolean> INDEX_QUERY_CACHE_EVERYTHING_SETTING =
         Setting.boolSetting("index.queries.cache.everything", false, Property.IndexScope);
 
-    // This setting is an escape hatch in case not caching term queries would slow some users down
-    // Do not document.
-    public static final Setting<Boolean> INDEX_QUERY_CACHE_TERM_QUERIES_SETTING =
-        Setting.boolSetting("index.queries.cache.term_queries", false, Property.IndexScope);
-
     private final IndexSettings indexSettings;
     private final AnalysisRegistry analysisRegistry;
     private final EngineFactory engineFactory;

core/src/main/java/org/elasticsearch/index/IndexingSlowLog.java

@@ -181,9 +181,9 @@ public String toString() {
             sb.append("type[").append(doc.type()).append("], ");
             sb.append("id[").append(doc.id()).append("], ");
             if (doc.routing() == null) {
-                sb.append("routing[] ");
+                sb.append("routing[]");
             } else {
-                sb.append("routing[").append(doc.routing()).append("] ");
+                sb.append("routing[").append(doc.routing()).append("]");
             }
 
             if (maxSourceCharsToLog == 0 || doc.source() == null || doc.source().length() == 0) {
@@ -193,7 +193,7 @@ public String toString() {
                 String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType());
                 sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog)).append("]");
             } catch (IOException e) {
-                sb.append(", source[_failed_to_convert_]");
+                sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]");
             }
             return sb.toString();
         }

core/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -290,11 +290,7 @@ public IndexShard(
         if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) {
             cachingPolicy = QueryCachingPolicy.ALWAYS_CACHE;
         } else {
-            QueryCachingPolicy cachingPolicy = new UsageTrackingQueryCachingPolicy();
-            if (IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING.get(settings) == false) {
-                cachingPolicy = new ElasticsearchQueryCachingPolicy(cachingPolicy);
-            }
-            this.cachingPolicy = cachingPolicy;
+            cachingPolicy = new UsageTrackingQueryCachingPolicy();
         }
         indexShardOperationPermits = new IndexShardOperationPermits(shardId, logger, threadPool);
         searcherWrapper = indexSearcherWrapper;