Merge remote-tracking branch 'elastic/ccr' into pr/31581

* elastic/ccr: (30 commits)
  Enable setting client path prefix to / (elastic#30119)
  [DOCS] Secure settings specified per node (elastic#31621)
  has_parent builder: exception message/param fix (elastic#31182)
  TEST: Randomize soft-deletes settings (elastic#31585)
  Mute 'Test typed keys parameter for suggesters' as we await a fix.
  Build test: Thread linger
  Fix gradle4.8 deprecation warnings (elastic#31654)
  Mute FileRealmTests#testAuthenticateCaching with an @AwaitsFix.
  Mute TransportChangePasswordActionTests#testIncorrectPasswordHashingAlgorithm with an @AwaitsFix.
  Build: Fix naming conventions task   (elastic#31681)
  Introduce a Hashing Processor (elastic#31087)
  Do not check for object existence when deleting repository index files (elastic#31680)
  Remove extra check for object existence in repository-gcs read object (elastic#31661)
  Support multiple system store types (elastic#31650)
  [Test] Clean up some repository-s3 tests (elastic#31601)
  [Docs] Use capital letters in section headings (elastic#31678)
  muted tests that will be replaced by the shard follow task refactoring: elastic#31581
  [DOCS] Add PQL language Plugin (elastic#31237)
  Merge AzureStorageService and AzureStorageServiceImpl and clean up tests (elastic#31607)
  TEST: Fix test task invocation (elastic#31657)
  ...

build.gradle

@@ -486,25 +486,17 @@ task run(type: Run) {
   impliesSubProjects = true
 }
 
-task wrapper(type: Wrapper)
-
-gradle.projectsEvaluated {
-
-  allprojects {
-    tasks.withType(Wrapper) { Wrapper wrapper ->
-      wrapper.distributionType = DistributionType.ALL
-
-      wrapper.doLast {
+wrapper {
+    distributionType = DistributionType.ALL
+    doLast {
         final DistributionLocator locator = new DistributionLocator()
         final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
         final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
         final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
         final String sha256Sum = new String(sha256Uri.toURL().bytes)
         wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
-      }
+        println "Added checksum to wrapper properties"
     }
-  }
-
 }
 
 static void assertLinesInFile(final Path path, final List<String> expectedLines) {
@@ -591,7 +583,7 @@ if (System.properties.get("build.compare") != null) {
       }
     }
     sourceBuild {
-      gradleVersion = "4.7" // does not default to gradle weapper of project dir, but current version
+      gradleVersion = "4.8.1" // does not default to gradle weapper of project dir, but current version
       projectDir = referenceProject
       tasks = ["clean", "assemble"]
       arguments = ["-Dbuild.compare_friendly=true"]

buildSrc/build.gradle

@@ -16,8 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-
 import java.nio.file.Files
 
 plugins {
@@ -41,6 +39,12 @@ if (project == rootProject) {
   buildDir = 'build-bootstrap'
 }
 
+// Make sure :buildSrc: doesn't generate classes incompatible with RUNTIME_JAVA_HOME
+// We can't use BuildPlugin here, so read from file
+String minimumRuntimeVersion = file('src/main/resources/minimumRuntimeVersion').text.trim()
+targetCompatibility = minimumRuntimeVersion
+sourceCompatibility = minimumRuntimeVersion
+
 /*****************************************************************************
  *         Propagating version.properties to the rest of the build           *
  *****************************************************************************/

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -19,7 +19,6 @@
 package org.elasticsearch.gradle
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
-import nebula.plugin.extraconfigurations.ProvidedBasePlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.eclipse.jgit.lib.Constants
 import org.eclipse.jgit.lib.RepositoryBuilder
@@ -58,9 +57,6 @@ import java.time.ZonedDateTime
  */
 class BuildPlugin implements Plugin<Project> {
 
-    static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
-    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_10
-
     @Override
     void apply(Project project) {
         if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
@@ -95,6 +91,12 @@ class BuildPlugin implements Plugin<Project> {
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
+            JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(
+                    BuildPlugin.class.getClassLoader().getResourceAsStream("minimumRuntimeVersion").text.trim()
+            )
+            JavaVersion minimumCompilerVersion = JavaVersion.toVersion(
+                    BuildPlugin.class.getClassLoader().getResourceAsStream("minimumCompilerVersion").text.trim()
+            )
             String compilerJavaHome = findCompilerJavaHome()
             String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
@@ -192,10 +194,12 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
             project.rootProject.ext.javaVersions = javaVersions
             project.rootProject.ext.buildChecksDone = true
+            project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
+            project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
         }
 
-        project.targetCompatibility = minimumRuntimeVersion
-        project.sourceCompatibility = minimumRuntimeVersion
+        project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
+        project.sourceCompatibility = project.rootProject.ext.minimumRuntimeVersion
 
         // set java home for each project, so they dont have to find it in the root project
         project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome
@@ -348,7 +352,7 @@ class BuildPlugin implements Plugin<Project> {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
             }
-            configuration.resolutionStrategy { 
+            configuration.resolutionStrategy {
                 failOnVersionConflict()
             }
         })
@@ -467,6 +471,24 @@ class BuildPlugin implements Plugin<Project> {
 
     /**Configuration generation of maven poms. */
     public static void configurePomGeneration(Project project) {
+        // Only works with  `enableFeaturePreview('STABLE_PUBLISHING')`
+        // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
+        project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
+            // The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it,
+            // just make a copy.
+            doLast {
+                project.copy {
+                    from generatePOMTask.destination
+                    into "${project.buildDir}/distributions"
+                    rename { "${project.archivesBaseName}-${project.version}.pom" }
+                }
+            }
+            // build poms with assemble (if the assemble task exists)
+            Task assemble = project.tasks.findByName('assemble')
+            if (assemble) {
+                assemble.dependsOn(generatePOMTask)
+            }
+        }
         project.plugins.withType(MavenPublishPlugin.class).whenPluginAdded {
             project.publishing {
                 publications {
@@ -476,20 +498,6 @@ class BuildPlugin implements Plugin<Project> {
                     }
                 }
             }
-
-            // Work around Gradle 4.8 issue until we `enableFeaturePreview('STABLE_PUBLISHING')`
-            // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
-            project.getGradle().getTaskGraph().whenReady {
-                project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
-                    // place the pom next to the jar it is for
-                    t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                    // build poms with assemble (if the assemble task exists)
-                    Task assemble = project.tasks.findByName('assemble')
-                    if (assemble) {
-                        assemble.dependsOn(t)
-                    }
-                }
-            }
         }
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -159,16 +159,18 @@ public class PluginBuildPlugin extends BuildPlugin {
     /** Adds a task to move jar and associated files to a "-client" name. */
     protected static void addClientJarTask(Project project) {
         Task clientJar = project.tasks.create('clientJar')
-        clientJar.dependsOn(project.jar, 'generatePomFileForClientJarPublication', project.javadocJar, project.sourcesJar)
+        clientJar.dependsOn(project.jar, project.tasks.generatePomFileForClientJarPublication, project.javadocJar, project.sourcesJar)
         clientJar.doFirst {
             Path jarFile = project.jar.outputs.files.singleFile.toPath()
             String clientFileName = jarFile.fileName.toString().replace(project.version, "client-${project.version}")
             Files.copy(jarFile, jarFile.resolveSibling(clientFileName), StandardCopyOption.REPLACE_EXISTING)
 
-            String pomFileName = jarFile.fileName.toString().replace('.jar', '.pom')
             String clientPomFileName = clientFileName.replace('.jar', '.pom')
-            Files.copy(jarFile.resolveSibling(pomFileName), jarFile.resolveSibling(clientPomFileName),
-                    StandardCopyOption.REPLACE_EXISTING)
+            Files.copy(
+                    project.tasks.generatePomFileForClientJarPublication.outputs.files.singleFile.toPath(),
+                    jarFile.resolveSibling(clientPomFileName),
+                    StandardCopyOption.REPLACE_EXISTING
+            )
 
             String sourcesFileName = jarFile.fileName.toString().replace('.jar', '-sources.jar')
             String clientSourcesFileName = clientFileName.replace('.jar', '-sources.jar')

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -29,6 +29,7 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
+import org.gradle.api.tasks.compile.JavaCompile
 
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
@@ -61,5 +62,12 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
 
         PrecommitTasks.create(project, false)
         project.check.dependsOn(project.precommit)
+
+        project.tasks.withType(JavaCompile) {
+            // This will be the default in Gradle 5.0
+            if (options.compilerArgs.contains("-processor") == false) {
+                options.compilerArgs << '-proc:none'
+            }
+        }
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy

@@ -50,12 +50,5 @@ public class StandaloneTestPlugin implements Plugin<Project> {
         test.testClassesDirs = project.sourceSets.test.output.classesDirs
         test.mustRunAfter(project.precommit)
         project.check.dependsOn(test)
-
-        project.tasks.withType(JavaCompile) {
-            // This will be the default in Gradle 5.0
-            if (options.compilerArgs.contains("-processor") == false) {
-                options.compilerArgs << '-proc:none'
-            }
-        }
     }
 }

buildSrc/src/main/resources/minimumCompilerVersion

@@ -0,0 +1 @@
+1.10

buildSrc/src/main/resources/minimumRuntimeVersion

@@ -0,0 +1 @@
+1.8

buildSrc/src/test/java/org/elasticsearch/gradle/test/BaseTestCase.java

@@ -21,6 +21,7 @@
 import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
 import com.carrotsearch.randomizedtesting.RandomizedRunner;
 import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
 import org.junit.Assert;
 import org.junit.runner.RunWith;
 
@@ -29,5 +30,6 @@
     JUnit4MethodProvider.class,
     JUnit3MethodProvider.class
 })
+@ThreadLeakLingering(linger = 5000) // wait for "Connection worker" to die
 public abstract class BaseTestCase extends Assert {
 }

client/benchmark/build.gradle

@@ -24,7 +24,7 @@ buildscript {
     }
   }
   dependencies {
-    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.2'
+    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
   }
 }
 
@@ -46,9 +46,7 @@ mainClassName = 'org.elasticsearch.client.benchmark.BenchmarkMain'
 
 
 // never try to invoke tests on the benchmark project - there aren't any
-check.dependsOn.remove(test)
-// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
-task test(type: Test, overwrite: true)
+test.enabled = false
 
 dependencies {
   compile 'org.apache.commons:commons-math3:3.2'

client/rest/src/main/java/org/elasticsearch/client/RestClient.java

@@ -794,8 +794,10 @@ static URI buildUri(String pathPrefix, String path, Map<String, String> params)
         Objects.requireNonNull(path, "path must not be null");
         try {
             String fullPath;
-            if (pathPrefix != null) {
-                if (path.startsWith("/")) {
+            if (pathPrefix != null && pathPrefix.isEmpty() == false) {
+                if (pathPrefix.endsWith("/") && path.startsWith("/")) {
+                    fullPath = pathPrefix.substring(0, pathPrefix.length() - 1) + path;
+                } else if (pathPrefix.endsWith("/") || path.startsWith("/")) {
                     fullPath = pathPrefix + path;
                 } else {
                     fullPath = pathPrefix + "/" + path;

client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java

@@ -143,32 +143,33 @@ public RestClientBuilder setRequestConfigCallback(RequestConfigCallback requestC
      * For example, if this is set to "/my/path", then any client request will become <code>"/my/path/" + endpoint</code>.
      * <p>
      * In essence, every request's {@code endpoint} is prefixed by this {@code pathPrefix}. The path prefix is useful for when
-     * Elasticsearch is behind a proxy that provides a base path; it is not intended for other purposes and it should not be supplied in
-     * other scenarios.
+     * Elasticsearch is behind a proxy that provides a base path or a proxy that requires all paths to start with '/';
+     * it is not intended for other purposes and it should not be supplied in other scenarios.
      *
      * @throws NullPointerException if {@code pathPrefix} is {@code null}.
-     * @throws IllegalArgumentException if {@code pathPrefix} is empty, only '/', or ends with more than one '/'.
+     * @throws IllegalArgumentException if {@code pathPrefix} is empty, or ends with more than one '/'.
      */
     public RestClientBuilder setPathPrefix(String pathPrefix) {
         Objects.requireNonNull(pathPrefix, "pathPrefix must not be null");
-        String cleanPathPrefix = pathPrefix;
 
+        if (pathPrefix.isEmpty()) {
+            throw new IllegalArgumentException("pathPrefix must not be empty");
+        }
+
+        String cleanPathPrefix = pathPrefix;
         if (cleanPathPrefix.startsWith("/") == false) {
             cleanPathPrefix = "/" + cleanPathPrefix;
         }
 
         // best effort to ensure that it looks like "/base/path" rather than "/base/path/"
-        if (cleanPathPrefix.endsWith("/")) {
+        if (cleanPathPrefix.endsWith("/") && cleanPathPrefix.length() > 1) {
             cleanPathPrefix = cleanPathPrefix.substring(0, cleanPathPrefix.length() - 1);
 
             if (cleanPathPrefix.endsWith("/")) {
                 throw new IllegalArgumentException("pathPrefix is malformed. too many trailing slashes: [" + pathPrefix + "]");
             }
         }
 
-        if (cleanPathPrefix.isEmpty() || "/".equals(cleanPathPrefix)) {
-            throw new IllegalArgumentException("pathPrefix must not be empty or '/': [" + pathPrefix + "]");
-        }
 
         this.pathPrefix = cleanPathPrefix;
         return this;

client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java

@@ -180,7 +180,6 @@ public void testSetPathPrefixNull() {
     }
 
     public void testSetPathPrefixEmpty() {
-        assertSetPathPrefixThrows("/");
         assertSetPathPrefixThrows("");
     }
 

client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java

@@ -223,12 +223,33 @@ public void onFailure(Exception exception) {
     }
 
     public void testBuildUriLeavesPathUntouched() {
+        final Map<String, String> emptyMap = Collections.emptyMap();
         {
-            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", Collections.<String, String>emptyMap());
+            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", emptyMap);
             assertEquals("/foo$bar/index/type/id", uri.getPath());
         }
         {
-            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", Collections.<String, String>emptyMap());
+            URI uri = RestClient.buildUri("/", "/*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri("/", "*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "*", emptyMap);
+            assertEquals("*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri("", "*", emptyMap);
+            assertEquals("*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", emptyMap);
             assertEquals("/foo$bar/ty/pe/i/d", uri.getPath());
         }
         {

docs/plugins/api.asciidoc

@@ -26,6 +26,9 @@ A number of plugins have been contributed by our community:
 * https://github.com/NLPchina/elasticsearch-sql/[SQL language Plugin]:
   Allows Elasticsearch to be queried with SQL (by nlpcn)
 
+* https://github.com/ritesh-kapoor/elasticsearch-pql[PQL language Plugin]:
+  Allows Elasticsearch to be queried with simple pipeline query syntax.
+
 * https://github.com/codelibs/elasticsearch-taste[Elasticsearch Taste Plugin]:
   Mahout Taste-based Collaborative Filtering implementation (by CodeLibs Project)
 

docs/reference/licensing/get-trial-status.asciidoc

@@ -13,12 +13,11 @@ This API enables you to check the status of your trial license.
 [float]
 ==== Description
 
-If a license is not already registered for the cluster, one is generated when
-the nodes start. By default, this is a 30-day trial license that gives access
-to all {xpack} features.
+If you want to try the features that are included in a platinum license, you can 
+start a 30-day trial. 
 
 NOTE: You are allowed to initiate a trial license only if your cluster has not
-already activated a trial license for the current major X-Pack version. For
+already activated a trial license for the current major product version. For
 example, if you have already activated a trial for v6.0, you cannot start a new
 trial until v7.0. You can, however, contact `info@elastic.co` to request an
 extended trial license.

docs/reference/licensing/start-basic.asciidoc

@@ -14,7 +14,7 @@ This API starts an indefinite basic license.
 ==== Description
 
 The `start basic` API enables you to initiate an indefinite basic license, which
-gives access to all {xpack} basic features. If the basic license does not support
+gives access to all the basic features. If the basic license does not support
 all of the features that are available with your current license, however, you are
 notified in the response.  You must then re-submit the API request with the
 `acknowledge` parameter set to `true`.