Merge branch 'master' into pr/26166

* master: (278 commits)
  Move pre-6.0 node checkpoint to SequenceNumbers
  Invalid JSON request body caused endless loop (elastic#26680)
  added comment
  fix line length violation
  Moved the check to fetch phase. This basically means that we throw a better error message instead of an AOBE and not adding more restrictions.
  inner hits: Do not allow inner hits that use _source and have a non nested object field as parent
  Separate Painless Whitelist Loading from the Painless Definition (elastic#26540)
  convert more admin requests to writeable (elastic#26566)
  Handle release of 5.6.1
  Allow `InputStreamStreamInput` array size validation where applicable (elastic#26692)
  Update global checkpoint with permit after recovery
  Filter pre-6.0 nodes for checkpoint invariants
  Skip bad request REST test on pre-6.0
  Reenable BWC tests after disabling for backport
  Add global checkpoint tracking on the primary
  [Test] Fix reference/cat/allocation/line_8 test failure
  [Docs] improved description for fs.total.available_in_bytes (elastic#26657)
  Fix discovery-file plugin to use custom config path
  fix testSniffNodes to use the new error message
  Add check for invalid index in WildcardExpressionResolver (elastic#26409)
  ...

CONTRIBUTING.md

@@ -91,6 +91,10 @@ Make sure you have [Gradle](http://gradle.org) installed, as
 Elasticsearch uses it as its build system. Gradle must be at least
 version 3.3 in order to build successfully.
 
+We support development in the Eclipse and IntelliJ IDEs. For Eclipse, the
+minimum version that we support is [Eclipse Oxygen][eclipse] (version 4.7). For
+IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
+
 Eclipse users can automatically configure their IDE: `gradle eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the
 option `Search for nested projects`. Additionally you will want to
@@ -102,11 +106,10 @@ then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable
 `Use auto-import`. Additionally, in order to run tests directly from 
-IDEA 2017.1 and above it is required to disable IDEA run launcher, 
+IDEA 2017.2 and above it is required to disable IDEA run launcher, 
 which can be achieved by adding `-Didea.no.launcher=true` 
 [JVM option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties) 
 
-
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This
 can be accomplished by passing the flag `-ea` to the JVM on startup.
@@ -172,3 +175,6 @@ especially when they are unlikely to become long time contributors.
 Finally, we require that you run `gradle check` before submitting a
 non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
+
+[eclipse]: http://www.eclipse.org/community/eclipse_newsletter/2017/june/
+[intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/

TESTING.asciidoc

@@ -548,3 +548,17 @@ included as part of the build by checking the projects of the build.
 ---------------------------------------------------------------------------
 gradle projects
 ---------------------------------------------------------------------------
+
+== Environment misc
+
+There is a known issue with macOS localhost resolve strategy that can cause
+some integration tests to fail. This is because integration tests have timings
+for cluster formation, discovery, etc. that can be exceeded if name resolution
+takes a long time.
+To fix this, make sure you have your computer name (as returned by `hostname`)
+inside `/etc/hosts`, e.g.:
+....
+127.0.0.1       localhost ElasticMBP.local
+255.255.255.255 broadcasthost
+::1             localhost ElasticMBP.local`
+....

build.gradle

@@ -23,6 +23,7 @@ import org.eclipse.jgit.lib.Repository
 import org.eclipse.jgit.lib.RepositoryBuilder
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.Version
 
@@ -60,6 +61,10 @@ configure(subprojects.findAll { it.projectDir.toPath().startsWith(rootPath) }) {
       }
     }
   }
+  plugins.withType(BuildPlugin).whenPluginAdded {
+    project.licenseFile = project.rootProject.file('LICENSE.txt')
+    project.noticeFile = project.rootProject.file('NOTICE.txt')
+  }
 }
 
 /* Introspect all versions of ES that may be tested agains for backwards
@@ -122,6 +127,16 @@ if (currentVersion.bugfix == 0) {
   }
 }
 
+// build metadata from previous build, contains eg hashes for bwc builds
+String buildMetadataValue = System.getenv('BUILD_METADATA')
+if (buildMetadataValue == null) {
+  buildMetadataValue = ''
+}
+Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectEntries {
+  def (String key, String value) = it.split('=')
+  return [key, value]
+}
+
 // injecting groovy property variables into all projects
 allprojects {
   project.ext {
@@ -131,6 +146,7 @@ allprojects {
     // for backcompat testing
     indexCompatVersions = versions
     wireCompatVersions = versions.subList(prevMinorIndex, versions.size())
+    buildMetadata = buildMetadataMap
   }
 }
 
@@ -188,33 +204,15 @@ task branchConsistency {
 }
 
 subprojects {
-  project.afterEvaluate {
-    // include license and notice in jars
-    tasks.withType(Jar) {
-      into('META-INF') {
-        from project.rootProject.rootDir
-        include 'LICENSE.txt'
-        include 'NOTICE.txt'
-      }
-    }
-    // ignore missing javadocs
-    tasks.withType(Javadoc) { Javadoc javadoc ->
-      // the -quiet here is because of a bug in gradle, in that adding a string option
-      // by itself is not added to the options. By adding quiet, both this option and
-      // the "value" -quiet is added, separated by a space. This is ok since the javadoc
-      // command already adds -quiet, so we are just duplicating it
-      // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
-      javadoc.options.encoding='UTF8'
-      javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
-      /*
-      TODO: building javadocs with java 9 b118 is currently broken with weird errors, so
-      for now this is commented out...try again with the next ea build...
-      javadoc.executable = new File(project.javaHome, 'bin/javadoc')
-      if (project.javaVersion == JavaVersion.VERSION_1_9) {
-        // TODO: remove this hack! gradle should be passing this...
-        javadoc.options.addStringOption('source', '8')
-      }*/
-    }
+  // ignore missing javadocs
+  tasks.withType(Javadoc) { Javadoc javadoc ->
+    // the -quiet here is because of a bug in gradle, in that adding a string option
+    // by itself is not added to the options. By adding quiet, both this option and
+    // the "value" -quiet is added, separated by a space. This is ok since the javadoc
+    // command already adds -quiet, so we are just duplicating it
+    // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
+    javadoc.options.encoding='UTF8'
+    javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
   }
 
   /* Sets up the dependencies that we build as part of this project but
@@ -272,6 +270,27 @@ subprojects {
         }
       }
     }
+
+    // Handle javadoc dependencies across projects. Order matters: the linksOffline for
+    // org.elasticsearch:elasticsearch must be the last one or all the links for the
+    // other packages (e.g org.elasticsearch.client) will point to core rather than
+    // their own artifacts.
+    if (project.plugins.hasPlugin(BuildPlugin)) {
+      String artifactsHost = VersionProperties.elasticsearch.endsWith("-SNAPSHOT") ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
+      Closure sortClosure = { a, b -> b.group <=> a.group }
+      Closure depJavadocClosure = { dep ->
+        if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
+          String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+          if (substitution != null) {
+            project.javadoc.dependsOn substitution + ':javadoc'
+            String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
+            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+          }
+        }
+      }
+      project.configurations.compile.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+      project.configurations.provided.dependencies.findAll().toSorted(sortClosure).each(depJavadocClosure)
+    }
   }
 }
 
@@ -281,7 +300,7 @@ subprojects {
 // the dependency is added.
 gradle.projectsEvaluated {
   allprojects {
-    if (project.path == ':test:framework' || project.path == ':client:test') {
+    if (project.path == ':test:framework') {
       // :test:framework:test cannot run before and after :core:test
       return
     }

buildSrc/build.gradle

@@ -94,6 +94,7 @@ dependencies {
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
   compile 'de.thetaphi:forbiddenapis:2.3'
   compile 'org.apache.rat:apache-rat:0.11'
+  compile "org.elasticsearch:jna:4.4.0-1"
 }
 
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -61,7 +61,7 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         ]
         RandomizedTestingTask newTestTask = tasks.create(properties)
         newTestTask.classpath = oldTestTask.classpath
-        newTestTask.testClassesDir = oldTestTask.testClassesDir
+        newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
 
         // hack so check task depends on custom test
         Task checkTask = tasks.findByPath('check')

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -30,25 +30,25 @@ import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.XmlProvider
 import org.gradle.api.artifacts.Configuration
-import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.ModuleDependency
 import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.file.CopySpec
 import org.gradle.api.plugins.JavaPlugin
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
+import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
 import org.gradle.util.GradleVersion
 
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
-
 /**
  * Encapsulates build configuration for elasticsearch projects.
  */
@@ -79,7 +79,7 @@ class BuildPlugin implements Plugin<Project> {
         configureConfigurations(project)
         project.ext.versions = VersionProperties.versions
         configureCompile(project)
-        configureJavadocJar(project)
+        configureJavadoc(project)
         configureSourcesJar(project)
         configurePomGeneration(project)
 
@@ -270,8 +270,8 @@ class BuildPlugin implements Plugin<Project> {
         })
 
         // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
-        Closure disableTransitiveDeps = { Dependency dep ->
-            if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency) && dep.group.startsWith('org.elasticsearch') == false) {
+        Closure disableTransitiveDeps = { ModuleDependency dep ->
+            if (!(dep instanceof ProjectDependency) && dep.group.startsWith('org.elasticsearch') == false) {
                 dep.transitive = false
 
                 // also create a configuration just for this dependency version, so that later
@@ -289,7 +289,7 @@ class BuildPlugin implements Plugin<Project> {
         project.configurations.provided.dependencies.all(disableTransitiveDeps)
     }
 
-    /** Adds repositores used by ES dependencies */
+    /** Adds repositories used by ES dependencies */
     static void configureRepositories(Project project) {
         RepositoryHandler repos = project.repositories
         if (System.getProperty("repos.mavenlocal") != null) {
@@ -454,6 +454,13 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    static void configureJavadoc(Project project) {
+        project.tasks.withType(Javadoc) {
+            executable = new File(project.javaHome, 'bin/javadoc')
+        }
+        configureJavadocJar(project)
+    }
+
     /** Adds a javadocJar task to generate a jar containing javadocs. */
     static void configureJavadocJar(Project project) {
         Jar javadocJarTask = project.task('javadocJar', type: Jar)
@@ -473,8 +480,10 @@ class BuildPlugin implements Plugin<Project> {
         project.assemble.dependsOn(sourcesJarTask)
     }
 
-    /** Adds additional manifest info to jars, and adds source and javadoc jars */
+    /** Adds additional manifest info to jars */
     static void configureJars(Project project) {
+        project.ext.licenseFile = null
+        project.ext.noticeFile = null
         project.tasks.withType(Jar) { Jar jarTask ->
             // we put all our distributable files under distributions
             jarTask.destinationDir = new File(project.buildDir, 'distributions')
@@ -498,6 +507,20 @@ class BuildPlugin implements Plugin<Project> {
                     jarTask.manifest.attributes('Change': 'Unknown')
                 }
             }
+            // add license/notice files
+            project.afterEvaluate {
+                if (project.licenseFile == null || project.noticeFile == null) {
+                    throw new GradleException("Must specify license and notice file for project ${project.path}")
+                }
+                jarTask.into('META-INF') {
+                    from(project.licenseFile.parent) {
+                        include project.licenseFile.name
+                    }
+                    from(project.noticeFile.parent) {
+                        include project.noticeFile.name
+                    }
+                }
+            }
         }
     }
 
@@ -526,8 +549,6 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.artifact', project.name
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
-            // Breaking change in JDK-9, revert to JDK-8 behavior for now, see https://github.com/elastic/elasticsearch/issues/21534
-            systemProperty 'jdk.io.permissionsUseCanonicalPath', 'true'
             systemProperty 'jna.nosys', 'true'
             // default test sysprop values
             systemProperty 'tests.ifNoTests', 'fail'

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -93,7 +93,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
          *
          * `sh` snippets that contain `curl` almost always should be marked
          * with `// CONSOLE`. In the exceptionally rare cases where they are
-         * not communicating with Elasticsearch, like the xamples in the ec2
+         * not communicating with Elasticsearch, like the examples in the ec2
          * and gce discovery plugins, the snippets should be marked
          * `// NOTCONSOLE`. */
         return snippet.language == 'js' || snippet.curl

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -19,6 +19,10 @@
 
 package org.elasticsearch.gradle.doc
 
+import groovy.json.JsonException
+import groovy.json.JsonParserType
+import groovy.json.JsonSlurper
+
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.ConfigurableFileTree
@@ -117,6 +121,23 @@ public class SnippetsTask extends DefaultTask {
                             + "contain `curl`.")
                     }
                 }
+                if (snippet.testResponse && snippet.language == 'js') {
+                    String quoted = snippet.contents
+                        // quote values starting with $
+                        .replaceAll(/([:,])\s*(\$[^ ,\n}]+)/, '$1 "$2"')
+                        // quote fields starting with $
+                        .replaceAll(/(\$[^ ,\n}]+)\s*:/, '"$1":')
+                    JsonSlurper slurper =
+                        new JsonSlurper(type: JsonParserType.INDEX_OVERLAY)
+                    try {
+                        slurper.parseText(quoted)
+                    } catch (JsonException e) {
+                        throw new InvalidUserDataException("Invalid json "
+                            + "in $snippet. The error is:\n${e.message}.\n"
+                            + "After substitutions and munging, the json "
+                            + "looks like:\n$quoted", e)
+                    }
+                }
                 perSnippet(snippet)
                 snippet = null
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -46,6 +46,10 @@ class PluginPropertiesExtension {
     @Input
     boolean hasClientJar = false
 
+    /** True if the plugin requires the elasticsearch keystore to exist, false otherwise. */
+    @Input
+    boolean requiresKeystore = false
+
     /** A license file that should be included in the built plugin zip. */
     @Input
     File licenseFile = null

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -80,7 +80,8 @@ class PluginPropertiesTask extends Copy {
             'elasticsearchVersion': stringSnap(VersionProperties.elasticsearch),
             'javaVersion': project.targetCompatibility as String,
             'classname': extension.classname,
-            'hasNativeController': extension.hasNativeController
+            'hasNativeController': extension.hasNativeController,
+            'requiresKeystore': extension.requiresKeystore
         ]
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy

@@ -63,11 +63,6 @@ public class ForbiddenPatternsTask extends DefaultTask {
         patterns.put('nocommit should be all lowercase or all uppercase',
             /((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
         patterns.put('tab', /\t/)
-        if (System.getProperty('build.snapshot', 'true').equals('false')) {
-            patterns.put('norelease', /norelease|NORELEASE/)
-        }
-        patterns.put('norelease should be all lowercase or all uppercase',
-            /((?i)norelease)(?<!(norelease|NORELEASE))/)
 
 
         inputs.property("excludes", filesFilter.excludes)