Merge branch 'master' into pr/24856

* master: (38 commits)
  Fix Lucene version expectation
  Verify Lucene version constants
  Avoid double decrement on current query counter
  Remove the need for _UNRELEASED suffix in versions (elastic#24798)
  Adjust available and free bytes to be non-negative on huge FSes
  Begin replacing static index tests with full restart tests (elastic#24846)
  Fix plugin docs for using custom config dir
  Update context-suggest.asciidoc
  Move BWC version to 5.5 after backport
  Support Multiple Collapse Inner Hits
  Scripting: Rename CompiledType to FactoryType in ScriptContext (elastic#24897)
  Scripting: Make contexts available to ScriptEngine construction (elastic#24896)
  Mute index and relocate concurrently
  Build: Add back explicit exclusions and remove gradle exclusions (elastic#24879)
  Scripting: Move context definitions to instance type classes (elastic#24883)
  Build: Fix hadoop integ test error on windows (elastic#24885)
  Put mapping and index template requests do not need content type detection for 5.3.0+ (elastic#24835)
  Add the ability to store objects with a ScrollContext (elastic#24777)
  add docs example for Ingest scripts manipulating document metadata (elastic#24875)
  Fix error message if an incompatible node connects (elastic#24884)
  ...

build.gradle

@@ -62,7 +62,12 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
   }
 }
 
-// introspect all versions of ES that may be tested agains for backwards compatibility
+/* Introspect all versions of ES that may be tested agains for backwards
+ * compatibility. It is *super* important that this logic is the same as the
+ * logic in VersionUtils.java, modulo alphas, betas, and rcs which are ignored
+ * in gradle because they don't have any backwards compatibility guarantees
+ * but are not ignored in VersionUtils.java because the tests expect them not
+ * to be. */
 Version currentVersion = Version.fromString(VersionProperties.elasticsearch.minus('-SNAPSHOT'))
 int prevMajor = currentVersion.major - 1
 File versionFile = file('core/src/main/java/org/elasticsearch/Version.java')
@@ -72,13 +77,14 @@ List<Version> versions = []
 int prevMinorIndex = -1 // index in the versions list of the last minor from the prev major
 int lastPrevMinor = -1 // the minor version number from the prev major we most recently seen
 for (String line : versionLines) {
-  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+)(_UNRELEASED)? .*/
+  /* Note that this skips alphas and betas which is fine because they aren't
+   * compatible with anything. */
+  Matcher match = line =~ /\W+public static final Version V_(\d+)_(\d+)_(\d+) .*/
   if (match.matches()) {
     int major = Integer.parseInt(match.group(1))
     int minor = Integer.parseInt(match.group(2))
     int bugfix = Integer.parseInt(match.group(3))
-    boolean unreleased = match.group(4) != null
-    Version foundVersion = new Version(major, minor, bugfix, false, unreleased)
+    Version foundVersion = new Version(major, minor, bugfix, false)
     if (currentVersion != foundVersion) {
       versions.add(foundVersion)
     }
@@ -96,10 +102,13 @@ if (currentVersion.bugfix == 0) {
   // If on a release branch, after the initial release of that branch, the bugfix version will
   // be bumped, and will be != 0. On master and N.x branches, we want to test against the
   // unreleased version of closest branch. So for those cases, the version includes -SNAPSHOT,
-  // and the bwc-zip distribution will checkout and build that version.
+  // and the bwc distribution will checkout and build that version.
   Version last = versions[-1]
-  versions[-1] = new Version(last.major, last.minor, last.bugfix,
-      true, last.unreleased)
+  versions[-1] = new Version(last.major, last.minor, last.bugfix, true)
+  if (last.bugfix == 0) {
+    versions[-2] = new Version(
+        versions[-2].major, versions[-2].minor, versions[-2].bugfix, true)
+  }
 }
 
 // injecting groovy property variables into all projects
@@ -114,6 +123,44 @@ allprojects {
   }
 }
 
+task('verifyVersions') {
+  description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
+  group 'Verification'
+  enabled = false == gradle.startParameter.isOffline()
+  doLast {
+    // Read the list from maven central
+    Node xml
+    new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+        xml = new XmlParser().parse(s)
+    }
+    Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })
+
+    // Limit the known versions to those that should be index compatible
+    knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }
+
+    /* Limit the listed versions to those that have been marked as released.
+     * Versions not marked as released don't get the same testing and we want
+     * to make sure that we flip all unreleased versions to released as soon
+     * as possible after release. */
+    Set<String> actualVersions = new TreeSet<>(
+      indexCompatVersions
+        .findAll { false == it.snapshot }
+        .collect { it.toString() })
+
+    // TODO this is almost certainly going to fail on 5.4 when we release 5.5.0
+
+    // Finally, compare!
+    if (!knownVersions.equals(actualVersions)) {
+      throw new GradleException("out-of-date versions\nActual  :" +
+        actualVersions + "\nExpected:" + knownVersions +
+        "; update Version.java")
+    }
+  }
+}
+task('precommit') {
+  dependsOn(verifyVersions)
+}
+
 subprojects {
   project.afterEvaluate {
     // include license and notice in jars
@@ -173,8 +220,10 @@ subprojects {
   ]
   if (wireCompatVersions[-1].snapshot) {
     // if the most previous version is a snapshot, we need to connect that version to the
-    // bwc-zip project which will checkout and build that snapshot version
-    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc-zip'
+    // bwc project which will checkout and build that snapshot version
+    ext.projectSubstitutions["org.elasticsearch.distribution.deb:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
+    ext.projectSubstitutions["org.elasticsearch.distribution.rpm:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
+    ext.projectSubstitutions["org.elasticsearch.distribution.zip:elasticsearch:${wireCompatVersions[-1]}"] = ':distribution:bwc'
   }
   project.afterEvaluate {
     configurations.all {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -311,7 +311,13 @@ class BuildPlugin implements Plugin<Project> {
     /**
      * Returns a closure which can be used with a MavenPom for fixing problems with gradle generated poms.
      *
-     * The current fixup is to set compile time deps back to compile from runtime (known issue with maven-publish plugin).
+     * <ul>
+     *     <li>Remove transitive dependencies. We currently exclude all artifacts explicitly instead of using wildcards
+     *         as Ivy incorrectly translates POMs with * excludes to Ivy XML with * excludes which results in the main artifact
+     *         being excluded as well (see https://issues.apache.org/jira/browse/IVY-1531). Note that Gradle 2.14+ automatically
+     *         translates non-transitive dependencies to * excludes. We should revisit this when upgrading Gradle.</li>
+     *     <li>Set compile time deps back to compile from runtime (known issue with maven-publish plugin)</li>
+     * </ul>
      */
     private static Closure fixupDependencies(Project project) {
         return { XmlProvider xml ->
@@ -321,15 +327,53 @@ class BuildPlugin implements Plugin<Project> {
                 return
             }
 
-            // fix deps incorrectly marked as runtime back to compile time deps
-            // see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
+            // check each dependency for any transitive deps
             for (Node depNode : depsNodes.get(0).children()) {
+                String groupId = depNode.get('groupId').get(0).text()
+                String artifactId = depNode.get('artifactId').get(0).text()
+                String version = depNode.get('version').get(0).text()
+
+                // fix deps incorrectly marked as runtime back to compile time deps
+                // see https://discuss.gradle.org/t/maven-publish-plugin-generated-pom-making-dependency-scope-runtime/7494/4
                 boolean isCompileDep = project.configurations.compile.allDependencies.find { dep ->
                     dep.name == depNode.artifactId.text()
                 }
                 if (depNode.scope.text() == 'runtime' && isCompileDep) {
                     depNode.scope*.value = 'compile'
                 }
+
+                // remove any exclusions added by gradle, they contain wildcards and systems like ivy have bugs with wildcards
+                // see https://github.com/elastic/elasticsearch/issues/24490
+                NodeList exclusionsNode = depNode.get('exclusions')
+                if (exclusionsNode.size() > 0) {
+                    depNode.remove(exclusionsNode.get(0))
+                }
+
+                // collect the transitive deps now that we know what this dependency is
+                String depConfig = transitiveDepConfigName(groupId, artifactId, version)
+                Configuration configuration = project.configurations.findByName(depConfig)
+                if (configuration == null) {
+                    continue // we did not make this dep non-transitive
+                }
+                Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
+                if (artifacts.size() <= 1) {
+                    // this dep has no transitive deps (or the only artifact is itself)
+                    continue
+                }
+
+                // we now know we have something to exclude, so add exclusions for all artifacts except the main one
+                Node exclusions = depNode.appendNode('exclusions')
+                for (ResolvedArtifact artifact : artifacts) {
+                    ModuleVersionIdentifier moduleVersionIdentifier = artifact.moduleVersion.id;
+                    String depGroupId = moduleVersionIdentifier.group
+                    String depArtifactId = moduleVersionIdentifier.name
+                    // add exclusions for all artifacts except the main one
+                    if (depGroupId != groupId || depArtifactId != artifactId) {
+                        Node exclusion = exclusions.appendNode('exclusion')
+                        exclusion.appendNode('groupId', depGroupId)
+                        exclusion.appendNode('artifactId', depArtifactId)
+                    }
+                }
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/Version.groovy

@@ -29,20 +29,14 @@ public class Version {
     final int bugfix
     final int id
     final boolean snapshot
-    /**
-     * Is the vesion listed as {@code _UNRELEASED} in Version.java.
-     */
-    final boolean unreleased
 
-    public Version(int major, int minor, int bugfix, boolean snapshot,
-            boolean unreleased) {
+    public Version(int major, int minor, int bugfix, boolean snapshot) {
         this.major = major
         this.minor = minor
         this.bugfix = bugfix
         this.snapshot = snapshot
         this.id = major * 100000 + minor * 1000 + bugfix * 10 +
             (snapshot ? 1 : 0)
-        this.unreleased = unreleased
     }
 
     public static Version fromString(String s) {
@@ -54,7 +48,7 @@ public class Version {
             bugfix = bugfix.split('-')[0]
         }
         return new Version(parts[0] as int, parts[1] as int, bugfix as int,
-            snapshot, false)
+            snapshot)
     }
 
     @Override

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -46,11 +46,11 @@ class ClusterConfiguration {
     int transportPort = 0
 
     /**
-     * An override of the data directory. This may only be used with a single node.
-     * The value is lazily evaluated at runtime as a String path.
+     * An override of the data directory. Input is the node number and output
+     * is the override data directory.
      */
     @Input
-    Object dataDir = null
+    Closure<String> dataDir = null
 
     /** Optional override of the cluster name. */
     @Input

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -111,10 +111,7 @@ class NodeInfo {
         homeDir = homeDir(baseDir, config.distribution, nodeVersion)
         confDir = confDir(baseDir, config.distribution, nodeVersion)
         if (config.dataDir != null) {
-            if (config.numNodes != 1) {
-                throw new IllegalArgumentException("Cannot set data dir for integ test with more than one node")
-            }
-            dataDir = config.dataDir
+            dataDir = "${config.dataDir(nodeNum)}"
         } else {
             dataDir = new File(homeDir, "data")
         }

core/build.gradle

@@ -277,43 +277,3 @@ if (isEclipse == false || project.path == ":core-tests") {
   check.dependsOn integTest
   integTest.mustRunAfter test
 }
-
-task('verifyVersions') {
-  description 'Verifies that all released versions that are indexed compatible are listed in Version.java.'
-  group 'Verification'
-  enabled = false == gradle.startParameter.isOffline()
-  doLast {
-    // Read the list from maven central
-    Node xml
-    new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
-        xml = new XmlParser().parse(s)
-    }
-    Set<String> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ })
-
-    // Limit the known versions to those that should be wire compatible
-    String currentVersion = versions.elasticsearch.minus('-SNAPSHOT')
-    int prevMajor = Integer.parseInt(currentVersion.split('\\.')[0]) - 1
-    if (prevMajor == 4) {
-      // 4 didn't exist, it was 2.
-      prevMajor = 2;
-    }
-    knownVersions = knownVersions.findAll { Integer.parseInt(it.split('\\.')[0]) >= prevMajor }
-
-    /* Limit the listed versions to those that have been marked as released.
-     * Versions not marked as released don't get the same testing and we want
-     * to make sure that we flip all unreleased versions to released as soon
-     * as possible after release. */
-    Set<String> actualVersions = new TreeSet<>(
-      indexCompatVersions
-        .findAll { false == it.unreleased }
-        .collect { it.toString() })
-
-    // Finally, compare!
-    if (!knownVersions.equals(actualVersions)) {
-      throw new GradleException("out-of-date versions\nActual  :" +
-        actualVersions + "\nExpected:" + knownVersions +
-        "; update Version.java")
-    }
-  }
-}
-check.dependsOn(verifyVersions)