Warn users if security is implicitly disabled (#71650)

* Warn users if security is implicitly disabled (#70114)

Elasticsearch has security features implicitly disabled by default for
Basic and Trial licenses, unless explicitly set in the configuration
file.
This may be good for onboarding, but it also lead to unintended insecure
 clusters.
 This change introduces clear warnings when security features are
 implicitly disabled.
 - a warning header in each REST response if security is implicitly
 disabled;
 - a log message during cluster boot.

distribution/docker/src/test/java/org/elasticsearch/docker/test/DockerYmlTestSuiteIT.java

@@ -10,7 +10,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -24,11 +23,8 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.CharBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Base64;
 
 public class DockerYmlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
@@ -130,22 +126,4 @@ protected String getProtocol() {
         }
         return "https";
     }
-
-    private static String basicAuthHeaderValue(String username, SecureString passwd) {
-        CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1);
-        byte[] charBytes = null;
-        try {
-            chars.put(username).put(':').put(passwd.getChars());
-            charBytes = CharArrays.toUtf8Bytes(chars.array());
-
-            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
-            String basicToken = Base64.getEncoder().encodeToString(charBytes);
-            return "Basic " + basicToken;
-        } finally {
-            Arrays.fill(chars.array(), (char) 0);
-            if (charBytes != null) {
-                Arrays.fill(charBytes, (byte) 0);
-            }
-        }
-    }
 }

docs/build.gradle

@@ -49,6 +49,7 @@ testClusters.integTest {
 
   // enable regexes in painless so our tests don't complain about example snippets that use them
   setting 'script.painless.regex.enabled', 'true'
+  setting 'xpack.security.enabled', 'false'
   setting 'path.repo', "${buildDir}/cluster/shared/repo"
   Closure configFile = {
     extraConfigFile it, file("src/test/cluster/config/$it")

docs/reference/rest-api/usage.asciidoc

@@ -48,15 +48,7 @@ GET /_xpack/usage
 {
   "security" : {
     "available" : true,
-    "enabled" : false,
-    "ssl" : {
-      "http" : {
-        "enabled" : false
-      },
-      "transport" : {
-        "enabled" : false
-      }
-    }
+    "enabled" : false
   },
   "monitoring" : {
     "available" : true,

modules/ingest-geoip/qa/file-based-update/build.gradle

@@ -12,6 +12,8 @@ apply plugin: 'elasticsearch.rest-test'
 testClusters.all {
   testDistribution = 'DEFAULT'
   setting 'resource.reload.interval.high', '100ms'
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
 }
 
 tasks.named("integTest").configure {

modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java

@@ -11,6 +11,9 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.ObjectPath;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
@@ -53,4 +56,12 @@ private static Map<String, Object> toMap(Response response) throws IOException {
         return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
     }
 
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
 }

plugins/examples/painless-whitelist/build.gradle

@@ -23,6 +23,7 @@ dependencies {
 
 testClusters.all {
   testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'false'
 }
 
 tasks.named("test").configure { enabled = false }

qa/ccs-rolling-upgrade-remote-cluster/build.gradle

@@ -35,12 +35,14 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       numberOfNodes = 2
       versions = [bwcVersionStr, project.version]
       setting 'cluster.remote.node.attr', 'gateway'
+      setting 'xpack.security.enabled', 'false'
     }
     "${baseName}-remote" {
       numberOfNodes = 3
       versions = [bwcVersionStr, project.version]
       firstNode.setting 'node.attr.gateway', 'true'
       lastNode.setting 'node.attr.gateway', 'true'
+      setting 'xpack.security.enabled', 'false'
     }
   }
 

qa/ccs-unavailable-clusters/build.gradle

@@ -10,6 +10,11 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'
 
+testClusters.matching { it.name == "integTest" }.configureEach {
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
+}
+
 dependencies {
   testImplementation project(":client:rest-high-level")
 }

qa/ccs-unavailable-clusters/src/test/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java

@@ -38,7 +38,9 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -327,4 +329,12 @@ private HighLevelClient(RestClient restClient) {
             super(restClient, (client) -> {}, Collections.emptyList());
         }
     }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
 }

qa/die-with-dignity/build.gradle

@@ -21,6 +21,8 @@ javaRestTest {
 
 testClusters.matching { it.name == "javaRestTest" }.configureEach {
   systemProperty "die.with.dignity.test", "whatever"
+  setting 'xpack.security.enabled', 'true'
+  user username: 'admin', password: 'admin-password', role: 'superuser'
 }
 
 tasks.named("test").configure {

qa/die-with-dignity/src/javaRestTest/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -10,7 +10,9 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.BufferedReader;
@@ -26,6 +28,7 @@
 import static org.hamcrest.Matchers.not;
 
 public class DieWithDignityIT extends ESRestTestCase {
+
     public void testDieWithDignity() throws Exception {
         expectThrows(
             IOException.class,
@@ -99,7 +102,9 @@ protected boolean preserveClusterUponCompletion() {
 
     @Override
     protected final Settings restClientSettings() {
+        String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray()));
         return Settings.builder().put(super.restClientSettings())
+            .put(ThreadContext.PREFIX + ".Authorization", token)
             // increase the timeout here to 90 seconds to handle long waits for a green
             // cluster health. the waits for green need to be longer than a minute to
             // account for delayed shards

qa/full-cluster-restart/build.gradle

@@ -27,6 +27,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
       setting 'indices.memory.shard_inactive_time', '60m'
       setting 'http.content_type.required', 'true'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 
@@ -60,4 +61,4 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
   tasks.register(bwcTaskName(bwcVersion)) {
     dependsOn tasks.named("${baseName}#upgradedClusterTest")
   }
-}
+}

qa/logging-config/build.gradle

@@ -11,6 +11,10 @@ apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.standalone-test'
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 testClusters.integTest {
   /**
    * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch

qa/mixed-cluster/build.gradle

@@ -38,6 +38,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       numberOfNodes = 4
 
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 

qa/multi-cluster-search/build.gradle

@@ -25,6 +25,7 @@ testClusters {
     'remote-cluster' {
         numberOfNodes = 2
         setting 'node.roles', '[data,ingest,master]'
+        setting 'xpack.security.enabled', 'false'
     }
 }
 
@@ -38,6 +39,7 @@ testClusters.matching { it.name == "mixedClusterTest"}.configureEach {
   setting 'cluster.remote.my_remote_cluster.seeds',
     { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
   setting 'cluster.remote.connections_per_cluster', '1'
+  setting 'xpack.security.enabled', 'false'
 }
 
 tasks.register("integTest") {

qa/remote-clusters/src/test/java/org/elasticsearch/cluster/remote/test/AbstractMultiClusterRemoteTestCase.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
@@ -26,11 +25,8 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
-import java.nio.CharBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Base64;
 import java.util.Collections;
 
 public abstract class AbstractMultiClusterRemoteTestCase extends ESRestTestCase {
@@ -150,24 +146,6 @@ protected String getProtocol() {
         return "https";
     }
 
-    private static String basicAuthHeaderValue(String username, SecureString passwd) {
-        CharBuffer chars = CharBuffer.allocate(username.length() + passwd.length() + 1);
-        byte[] charBytes = null;
-        try {
-            chars.put(username).put(':').put(passwd.getChars());
-            charBytes = CharArrays.toUtf8Bytes(chars.array());
-
-            //TODO we still have passwords in Strings in headers. Maybe we can look into using a CharSequence?
-            String basicToken = Base64.getEncoder().encodeToString(charBytes);
-            return "Basic " + basicToken;
-        } finally {
-            Arrays.fill(chars.array(), (char) 0);
-            if (charBytes != null) {
-                Arrays.fill(charBytes, (byte) 0);
-            }
-        }
-    }
-
     private String getProperty(String key) {
         String value = System.getProperty(key);
         if (value == null) {

qa/repository-multi-version/build.gradle

@@ -30,6 +30,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
       version = v
       numberOfNodes = 2
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
     }
   }
 
@@ -76,4 +77,4 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
   tasks.register(bwcTaskName(bwcVersion)) {
     dependsOn tasks.named("${baseName}#Step4NewClusterTest")
   }
-}
+}

qa/rolling-upgrade/build.gradle

@@ -39,6 +39,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
 
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
       setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+      setting 'xpack.security.enabled', 'false'
       setting 'http.content_type.required', 'true'
     }
   }

qa/smoke-test-http/build.gradle

@@ -16,6 +16,10 @@ dependencies {
   testImplementation project(path: ':plugins:transport-nio') // for http
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 tasks.named("integTest").configure {
   /*
    * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each

qa/smoke-test-ingest-disabled/build.gradle

@@ -14,6 +14,9 @@ apply plugin: 'elasticsearch.rest-resources'
 dependencies {
   testImplementation project(':modules:ingest-common')
 }
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
 
 testClusters.integTest {
   setting 'node.roles', '[data,master,remote_cluster_client]'

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -19,6 +19,10 @@ dependencies {
   testImplementation project(':modules:reindex')
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 tasks.named("testingConventions").configure {
   naming {
     IT {

qa/smoke-test-multinode/build.gradle

@@ -23,6 +23,10 @@ testClusters.integTest {
   setting 'path.repo', repo.absolutePath
 }
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 integTest {
   doFirst {
     project.delete(repo)

qa/smoke-test-plugins/build.gradle

@@ -27,6 +27,7 @@ testClusters.matching { it.name == "integTest" }.configureEach {
   pluginPaths.each { pluginPath ->
     plugin pluginPath
   }
+  setting 'xpack.security.enabled', 'false'
 }
 
 ext.expansions = [

qa/unconfigured-node-name/build.gradle

@@ -12,6 +12,10 @@ apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 
+testClusters.all {
+  setting 'xpack.security.enabled', 'false'
+}
+
 testClusters.integTest {
   nameCustomization = { null }
 }

qa/verify-version-constants/build.gradle

@@ -22,6 +22,8 @@ for (Version bwcVersion : BuildParams.bwcVersions.indexCompatible) {
     "${baseName}" {
       version = bwcVersion.toString()
       setting 'http.content_type.required', 'true'
+      setting 'xpack.security.enabled', 'true'
+      user username: 'admin', password: 'admin-password', role: 'superuser'
     }
   }