Merge remote-tracking branch 'elastic/6.x' into ccr-6.x

* elastic/6.x:
  Enable setting client path prefix to / (#30119)
  [DOCS] Secure settings specified per node (#31621)
  Build test: Thread linger
  Build: Fix naming conventions task   (#31681)
  Introduce a Hashing Processor (#31087)

buildSrc/build.gradle

@@ -16,8 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
-
 import java.nio.file.Files
 
 plugins {
@@ -41,6 +39,12 @@ if (project == rootProject) {
   buildDir = 'build-bootstrap'
 }
 
+// Make sure :buildSrc: doesn't generate classes incompatible with RUNTIME_JAVA_HOME
+// We can't use BuildPlugin here, so read from file
+String minimumRuntimeVersion = file('src/main/resources/minimumRuntimeVersion').text.trim()
+targetCompatibility = minimumRuntimeVersion
+sourceCompatibility = minimumRuntimeVersion
+
 /*****************************************************************************
  *         Propagating version.properties to the rest of the build           *
  *****************************************************************************/

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -19,7 +19,6 @@
 package org.elasticsearch.gradle
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
-import nebula.plugin.extraconfigurations.ProvidedBasePlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.eclipse.jgit.lib.Constants
 import org.eclipse.jgit.lib.RepositoryBuilder
@@ -58,9 +57,6 @@ import java.time.ZonedDateTime
  */
 class BuildPlugin implements Plugin<Project> {
 
-    static final JavaVersion minimumRuntimeVersion = JavaVersion.VERSION_1_8
-    static final JavaVersion minimumCompilerVersion = JavaVersion.VERSION_1_10
-
     @Override
     void apply(Project project) {
         if (project.pluginManager.hasPlugin('elasticsearch.standalone-rest-test')) {
@@ -95,6 +91,12 @@ class BuildPlugin implements Plugin<Project> {
     /** Performs checks on the build environment and prints information about the build environment. */
     static void globalBuildInfo(Project project) {
         if (project.rootProject.ext.has('buildChecksDone') == false) {
+            JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(
+                    BuildPlugin.class.getClassLoader().getResourceAsStream("minimumRuntimeVersion").text.trim()
+            )
+            JavaVersion minimumCompilerVersion = JavaVersion.toVersion(
+                    BuildPlugin.class.getClassLoader().getResourceAsStream("minimumCompilerVersion").text.trim()
+            )
             String compilerJavaHome = findCompilerJavaHome()
             String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
@@ -192,10 +194,12 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
             project.rootProject.ext.javaVersions = javaVersions
             project.rootProject.ext.buildChecksDone = true
+            project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
+            project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
         }
 
-        project.targetCompatibility = minimumRuntimeVersion
-        project.sourceCompatibility = minimumRuntimeVersion
+        project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
+        project.sourceCompatibility = project.rootProject.ext.minimumRuntimeVersion
 
         // set java home for each project, so they dont have to find it in the root project
         project.ext.compilerJavaHome = project.rootProject.ext.compilerJavaHome
@@ -348,7 +352,7 @@ class BuildPlugin implements Plugin<Project> {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
             }
-            configuration.resolutionStrategy { 
+            configuration.resolutionStrategy {
                 failOnVersionConflict()
             }
         })

buildSrc/src/main/resources/minimumCompilerVersion

@@ -0,0 +1 @@
+1.10

buildSrc/src/main/resources/minimumRuntimeVersion

@@ -0,0 +1 @@
+1.8

buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java

@@ -4,7 +4,6 @@
 import org.gradle.testkit.runner.BuildResult;
 import org.gradle.testkit.runner.GradleRunner;
 import org.gradle.testkit.runner.TaskOutcome;
-import org.junit.Ignore;
 
 import java.util.Arrays;
 
@@ -21,7 +20,6 @@ public void testPluginCanBeApplied() {
         assertTrue(result.getOutput().contains("build plugin can be applied"));
     }
 
-    @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665")
     public void testNameCheckFailsAsItShould() {
         BuildResult result = GradleRunner.create()
             .withProjectDir(getProjectDir("namingConventionsSelfTest"))
@@ -46,7 +44,6 @@ public void testNameCheckFailsAsItShould() {
         }
     }
 
-    @Ignore("AwaitsFix : https://github.com/elastic/elasticsearch/issues/31665")
     public void testNameCheckFailsAsItShouldWithMain() {
         BuildResult result = GradleRunner.create()
             .withProjectDir(getProjectDir("namingConventionsSelfTest"))

buildSrc/src/test/java/org/elasticsearch/gradle/test/BaseTestCase.java

@@ -21,6 +21,7 @@
 import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
 import com.carrotsearch.randomizedtesting.RandomizedRunner;
 import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
 import org.junit.Assert;
 import org.junit.runner.RunWith;
 
@@ -29,5 +30,6 @@
     JUnit4MethodProvider.class,
     JUnit3MethodProvider.class
 })
+@ThreadLeakLingering(linger = 5000) // wait for "Connection worker" to die
 public abstract class BaseTestCase extends Assert {
 }

client/rest/src/main/java/org/elasticsearch/client/RestClient.java

@@ -794,8 +794,10 @@ static URI buildUri(String pathPrefix, String path, Map<String, String> params)
         Objects.requireNonNull(path, "path must not be null");
         try {
             String fullPath;
-            if (pathPrefix != null) {
-                if (path.startsWith("/")) {
+            if (pathPrefix != null && pathPrefix.isEmpty() == false) {
+                if (pathPrefix.endsWith("/") && path.startsWith("/")) {
+                    fullPath = pathPrefix.substring(0, pathPrefix.length() - 1) + path;
+                } else if (pathPrefix.endsWith("/") || path.startsWith("/")) {
                     fullPath = pathPrefix + path;
                 } else {
                     fullPath = pathPrefix + "/" + path;

client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java

@@ -143,32 +143,33 @@ public RestClientBuilder setRequestConfigCallback(RequestConfigCallback requestC
      * For example, if this is set to "/my/path", then any client request will become <code>"/my/path/" + endpoint</code>.
      * <p>
      * In essence, every request's {@code endpoint} is prefixed by this {@code pathPrefix}. The path prefix is useful for when
-     * Elasticsearch is behind a proxy that provides a base path; it is not intended for other purposes and it should not be supplied in
-     * other scenarios.
+     * Elasticsearch is behind a proxy that provides a base path or a proxy that requires all paths to start with '/';
+     * it is not intended for other purposes and it should not be supplied in other scenarios.
      *
      * @throws NullPointerException if {@code pathPrefix} is {@code null}.
-     * @throws IllegalArgumentException if {@code pathPrefix} is empty, only '/', or ends with more than one '/'.
+     * @throws IllegalArgumentException if {@code pathPrefix} is empty, or ends with more than one '/'.
      */
     public RestClientBuilder setPathPrefix(String pathPrefix) {
         Objects.requireNonNull(pathPrefix, "pathPrefix must not be null");
-        String cleanPathPrefix = pathPrefix;
 
+        if (pathPrefix.isEmpty()) {
+            throw new IllegalArgumentException("pathPrefix must not be empty");
+        }
+
+        String cleanPathPrefix = pathPrefix;
         if (cleanPathPrefix.startsWith("/") == false) {
             cleanPathPrefix = "/" + cleanPathPrefix;
         }
 
         // best effort to ensure that it looks like "/base/path" rather than "/base/path/"
-        if (cleanPathPrefix.endsWith("/")) {
+        if (cleanPathPrefix.endsWith("/") && cleanPathPrefix.length() > 1) {
             cleanPathPrefix = cleanPathPrefix.substring(0, cleanPathPrefix.length() - 1);
 
             if (cleanPathPrefix.endsWith("/")) {
                 throw new IllegalArgumentException("pathPrefix is malformed. too many trailing slashes: [" + pathPrefix + "]");
             }
         }
 
-        if (cleanPathPrefix.isEmpty() || "/".equals(cleanPathPrefix)) {
-            throw new IllegalArgumentException("pathPrefix must not be empty or '/': [" + pathPrefix + "]");
-        }
 
         this.pathPrefix = cleanPathPrefix;
         return this;

client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java

@@ -180,7 +180,6 @@ public void testSetPathPrefixNull() {
     }
 
     public void testSetPathPrefixEmpty() {
-        assertSetPathPrefixThrows("/");
         assertSetPathPrefixThrows("");
     }
 

client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java

@@ -223,12 +223,33 @@ public void onFailure(Exception exception) {
     }
 
     public void testBuildUriLeavesPathUntouched() {
+        final Map<String, String> emptyMap = Collections.emptyMap();
         {
-            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", Collections.<String, String>emptyMap());
+            URI uri = RestClient.buildUri("/foo$bar", "/index/type/id", emptyMap);
             assertEquals("/foo$bar/index/type/id", uri.getPath());
         }
         {
-            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", Collections.<String, String>emptyMap());
+            URI uri = RestClient.buildUri("/", "/*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri("/", "*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "*", emptyMap);
+            assertEquals("*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri("", "*", emptyMap);
+            assertEquals("*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/*", emptyMap);
+            assertEquals("/*", uri.getPath());
+        }
+        {
+            URI uri = RestClient.buildUri(null, "/foo$bar/ty/pe/i/d", emptyMap);
             assertEquals("/foo$bar/ty/pe/i/d", uri.getPath());
         }
         {

docs/reference/setup/secure-settings.asciidoc

@@ -1,5 +1,5 @@
 [[secure-settings]]
-=== Secure Settings
+=== Secure settings
 
 Some settings are sensitive, and relying on filesystem permissions to protect
 their values is not sufficient. For this use case, Elasticsearch provides a
@@ -16,6 +16,10 @@ Elasticsearch.
 NOTE: The elasticsearch keystore currently only provides obfuscation. In the future,
 password protection will be added.
 
+These settings, just like the regular ones in the `elasticsearch.yml` config file,
+need to be specified on each node in the cluster. Currently, all secure settings
+are node-specific settings that must have the same value on every node.
+
 [float]
 [[creating-keystore]]
 === Creating the keystore

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -90,6 +90,7 @@ public abstract class ESRestTestCase extends ESTestCase {
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
     public static final String CLIENT_RETRY_TIMEOUT = "client.retry.timeout";
     public static final String CLIENT_SOCKET_TIMEOUT = "client.socket.timeout";
+    public static final String CLIENT_PATH_PREFIX = "client.path.prefix";
 
     /**
      * Convert the entity from a {@link Response} into a map of maps.
@@ -383,7 +384,11 @@ private void waitForClusterStateUpdatesToFinish() throws Exception {
      * Used to obtain settings for the REST client that is used to send REST requests.
      */
     protected Settings restClientSettings() {
-        return Settings.EMPTY;
+        Settings.Builder builder = Settings.builder();
+        if (System.getProperty("tests.rest.client_path_prefix") != null) {
+            builder.put(CLIENT_PATH_PREFIX, System.getProperty("tests.rest.client_path_prefix"));
+        }
+        return builder.build();
     }
 
     /**
@@ -454,6 +459,9 @@ protected static void configureClient(RestClientBuilder builder, Settings settin
             final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT);
             builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis())));
         }
+        if (settings.hasValue(CLIENT_PATH_PREFIX)) {
+            builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
+        }
     }
 
     @SuppressWarnings("unchecked")

x-pack/plugin/build.gradle

@@ -152,6 +152,7 @@ integTestCluster {
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
   keystoreSetting 'xpack.security.authc.token.passphrase', 'x-pack-token-service-password'
   keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+  keystoreSetting 'xpack.security.ingest.hash.processor.key', 'hmackey'
   distribution = 'zip' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -176,6 +176,7 @@
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+import org.elasticsearch.xpack.security.ingest.HashProcessor;
 import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor;
 import org.elasticsearch.xpack.security.rest.SecurityRestFilter;
 import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction;
@@ -600,6 +601,8 @@ public static List<Setting<?>> getSettings(boolean transportClientMode, List<Sec
         settingsList.add(Setting.listSetting(SecurityField.setting("hide_settings"), Collections.emptyList(), Function.identity(),
                 Property.NodeScope, Property.Filtered));
         settingsList.add(INDICES_ADMIN_FILTERED_FIELDS_SETTING);
+        // ingest processor settings
+        settingsList.add(HashProcessor.HMAC_KEY_SETTING);
 
         return settingsList;
     }
@@ -744,7 +747,10 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        return Collections.singletonMap(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
+        Map<String, Processor.Factory> processors = new HashMap<>();
+        processors.put(SetSecurityUserProcessor.TYPE, new SetSecurityUserProcessor.Factory(parameters.threadContext));
+        processors.put(HashProcessor.TYPE, new HashProcessor.Factory(parameters.env.settings()));
+        return processors;
     }
 
     /**