Skip to content

Commit

Permalink
Add build utility to check cluster health over ssl (#40573)
Browse files Browse the repository at this point in the history
By default, in integ tests we wait for the standalone cluster to start
by using the ant Get task to retrieve the cluster health endpoint.
However the ant task has no facilities for customising the trusted
CAs for a https resource, so if the integ test cluster has TLS enabled
on the http interface (using a custom CA) we need a separate utility
for that purpose.

Resolves: #38072
  • Loading branch information
tvernum authored Apr 1, 2019
1 parent b3341da commit c652917
Show file tree
Hide file tree
Showing 7 changed files with 333 additions and 97 deletions.
1 change: 1 addition & 0 deletions buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ if (project != rootProject) {

forbiddenPatterns {
exclude '**/*.wav'
exclude '**/*.p12'
// the file that actually defines nocommit
exclude '**/ForbiddenPatternsTask.java'
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle.http;

import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;

import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.KeyManager;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.SecureRandom;
import java.security.cert.Certificate;
import java.security.cert.CertificateFactory;
import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;

/**
* A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
* This is logically similar to using the Ant Get task to retrieve a resource, but with the difference that it can
* access resources that do not use the JRE's default trusted CAs.
*/
public class WaitForHttpResource {

private static final Logger logger = Logging.getLogger(WaitForHttpResource.class);

private Set<Integer> validResponseCodes = Collections.singleton(200);
private URL url;
private Set<File> certificateAuthorities;
private File trustStoreFile;
private String trustStorePassword;
private String username;
private String password;

public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
}

public WaitForHttpResource(URL url) {
this.url = url;
}

public void setValidResponseCodes(int... validResponseCodes) {
this.validResponseCodes = new HashSet<>(validResponseCodes.length);
for (int rc : validResponseCodes) {
this.validResponseCodes.add(rc);
}
}

public void setCertificateAuthorities(File... certificateAuthorities) {
this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
}

public void setTrustStoreFile(File trustStoreFile) {
this.trustStoreFile = trustStoreFile;
}

public void setTrustStorePassword(String trustStorePassword) {
this.trustStorePassword = trustStorePassword;
}

public void setUsername(String username) {
this.username = username;
}

public void setPassword(String password) {
this.password = password;
}

public boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
final long sleep = Long.max(durationInMs / 10, 100);

final SSLContext ssl;
final KeyStore trustStore = buildTrustStore();
if (trustStore != null) {
ssl = createSslContext(trustStore);
} else {
ssl = null;
}
IOException failure = null;
for (; ; ) {
try {
checkResource(ssl);
return true;
} catch (IOException e) {
logger.debug("Failed to access resource [{}]", url, e);
failure = e;
}
if (System.nanoTime() < waitUntil) {
Thread.sleep(sleep);
} else {
logger.error("Failed to access url [{}]", url, failure);
return false;
}
}
}

protected void checkResource(SSLContext ssl) throws IOException {
try {
final HttpURLConnection connection = buildConnection(ssl);
connection.connect();
final Integer response = connection.getResponseCode();
if (validResponseCodes.contains(response)) {
logger.info("Got successful response [{}] from URL [{}]", response, url);
return;
} else {
throw new IOException(response + " " + connection.getResponseMessage());
}
} catch (IOException e) {
throw e;
}
}

HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
configureSslContext(connection, ssl);
configureBasicAuth(connection);
connection.setRequestMethod("GET");
return connection;
}

private void configureSslContext(HttpURLConnection connection, SSLContext ssl) {
if (ssl != null) {
if (connection instanceof HttpsURLConnection) {
((HttpsURLConnection) connection).setSSLSocketFactory(ssl.getSocketFactory());
} else {
throw new IllegalStateException("SSL trust has been configured, but [" + url + "] is not a 'https' URL");
}
}
}

private void configureBasicAuth(HttpURLConnection connection) {
if (username != null) {
if (password == null) {
throw new IllegalStateException("Basic Auth user [" + username
+ "] has been set, but no password has been configured");
}
connection.setRequestProperty("Authorization",
"Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)));
}
}

KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
if (this.certificateAuthorities != null) {
if (trustStoreFile != null) {
throw new IllegalStateException("Cannot specify both truststore and CAs");
}
return buildTrustStoreFromCA();
} else if (trustStoreFile != null) {
return buildTrustStoreFromFile();
} else {
return null;
}
}

private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
try (InputStream input = new FileInputStream(trustStoreFile)) {
keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
}
return keyStore;
}

private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
store.load(null, null);
final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
int counter = 0;
for (File ca : certificateAuthorities) {
try (InputStream input = new FileInputStream(ca)) {
for (Certificate certificate : certFactory.generateCertificates(input)) {
store.setCertificateEntry("cert-" + counter, certificate);
counter++;
}
}
}
return store;
}

private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
checkForTrustEntry(trustStore);
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
return sslContext;
}

private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
Enumeration<String> enumeration = trustStore.aliases();
while (enumeration.hasMoreElements()) {
if (trustStore.isCertificateEntry(enumeration.nextElement())) {
// found trusted cert entry
return;
}
}
throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.elasticsearch.gradle.http;

import org.elasticsearch.gradle.test.GradleUnitTestCase;

import java.io.File;
import java.net.URL;
import java.security.KeyStore;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;

import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.notNullValue;

public class WaitForHttpResourceTests extends GradleUnitTestCase {

public void testBuildTrustStoreFromFile() throws Exception {
final WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"));
final URL ca = getClass().getResource("/ca.p12");
assertThat(ca, notNullValue());
http.setTrustStoreFile(new File(ca.getPath()));
http.setTrustStorePassword("password");
final KeyStore store = http.buildTrustStore();
final Certificate certificate = store.getCertificate("ca");
assertThat(certificate, notNullValue());
assertThat(certificate, instanceOf(X509Certificate.class));
assertThat(((X509Certificate)certificate).getSubjectDN().toString(), equalTo("CN=Elastic Certificate Tool Autogenerated CA"));
}

public void testBuildTrustStoreFromCA() throws Exception {
final WaitForHttpResource http = new WaitForHttpResource(new URL("https://localhost/"));
final URL ca = getClass().getResource("/ca.pem");
assertThat(ca, notNullValue());
http.setCertificateAuthorities(new File(ca.getPath()));
final KeyStore store = http.buildTrustStore();
final Certificate certificate = store.getCertificate("cert-0");
assertThat(certificate, notNullValue());
assertThat(certificate, instanceOf(X509Certificate.class));
assertThat(((X509Certificate)certificate).getSubjectDN().toString(), equalTo("CN=Elastic Certificate Tool Autogenerated CA"));
}
}
Binary file added buildSrc/src/test/resources/ca.p12
Binary file not shown.
25 changes: 25 additions & 0 deletions buildSrc/src/test/resources/ca.pem
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
Bag Attributes
friendlyName: ca
localKeyID: 54 69 6D 65 20 31 35 35 33 37 34 33 38 39 30 38 33 35
subject=/CN=Elastic Certificate Tool Autogenerated CA
issuer=/CN=Elastic Certificate Tool Autogenerated CA
-----BEGIN CERTIFICATE-----
MIIDSjCCAjKgAwIBAgIVAMQMmDRcXfXLaTp6ep1H8rC3tOrwMA0GCSqGSIb3DQEB
CwUAMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlmaWNhdGUgVG9vbCBBdXRvZ2Vu
ZXJhdGVkIENBMB4XDTE5MDMyODAzMzEyNloXDTIyMDMyNzAzMzEyNlowNDEyMDAG
A1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5lcmF0ZWQgQ0Ew
ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDT73N6JZeBPyzahc0aNcra
BpUROVGB9wXQqf8JeU4GtH+1qfqUKYKUJTe/DZWc+5Qz1WAKGZEvBySAlgbuncuq
VpLzWxpEui1vRW8JB3gjZgeY3vfErrEWWr95YM0e8rWu4AoAchzqsrG0/+po2eui
cN+8hI6jRKiBv/ZeQqja6KZ8y4Wt4VaNVL53+I7+eWA/aposu6/piUg2wZ/FNhVK
hypcJwDdp3fQaugtPj3y76303jTRgutgd3rtWFuy3MCDLfs3mSQUjO10s93zwLdC
XokyIywijS5CpO8mEuDRu9rb5J1DzwUpUfk+GMObb6rHjFKzSqnM3s+nasypQQ9L
AgMBAAGjUzBRMB0GA1UdDgQWBBQZEW88R95zSzO2tLseEWgI7ugvLzAfBgNVHSME
GDAWgBQZEW88R95zSzO2tLseEWgI7ugvLzAPBgNVHRMBAf8EBTADAQH/MA0GCSqG
SIb3DQEBCwUAA4IBAQBEJN0UbL77usVnzIvxKa3GpLBgJQAZtD1ifZppC4w46Bul
1G7Fdc+XMbzZlI4K6cWEdd5dfEssKA8btEtRzdNOqgggBpqrUU0mNlQ+vC22XORU
ykHAu2TsRwoHmuxkd9Et/QyuTFXR4fTiU8rsJuLFOgn+RdEblA0J0gJeIqdWI5Z1
z13OyZEl6BCQFyrntu2eERxaHEfsJOSBZE4RcecnLNGhIJBXE0Pk4iTiViJF/h7d
+kUUegKx0qewZif2eEZgrz12Vuen9a6bh2i2pNS95vABVVMr8uB+J1BGkNA5YT7J
qtZA2tN//Evng7YDiR+KkB1kvXVZVIi2WPDLD/zu
-----END CERTIFICATE-----
57 changes: 7 additions & 50 deletions x-pack/qa/reindex-tests-with-security/build.gradle
Original file line number Diff line number Diff line change
@@ -1,10 +1,4 @@
import javax.net.ssl.HttpsURLConnection
import javax.net.ssl.KeyManager
import javax.net.ssl.SSLContext
import javax.net.ssl.TrustManagerFactory
import java.nio.charset.StandardCharsets
import java.security.KeyStore
import java.security.SecureRandom
import org.elasticsearch.gradle.http.WaitForHttpResource

apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
Expand Down Expand Up @@ -57,48 +51,11 @@ integTestCluster {
'bin/elasticsearch-users', 'useradd', user, '-p', 'x-pack-test-password', '-r', role
}
waitCondition = { node, ant ->
// Load the CA PKCS#12 file as a truststore
KeyStore ks = KeyStore.getInstance("PKCS12");
ks.load(caFile.newInputStream(), 'password'.toCharArray());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(ks);

// Configre a SSL context for TLS1.2 using our CA trust manager
SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());

// Check whether the cluster has started
URL url = new URL("https://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow");
for (int i = 20; i >= 0; i--) {
// we use custom wait logic here for HTTPS
HttpsURLConnection httpURLConnection = null;
try {
logger.info("Trying ${url}");
httpURLConnection = (HttpsURLConnection) url.openConnection();
httpURLConnection.setSSLSocketFactory(sslContext.getSocketFactory());
httpURLConnection.setRequestProperty("Authorization",
"Basic " + Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
httpURLConnection.setRequestMethod("GET");
httpURLConnection.connect();
if (httpURLConnection.getResponseCode() == 200) {
logger.info("Cluster has started");
return true;
} else {
logger.debug("HTTP response was [{}]", httpURLConnection.getResponseCode());
}
} catch (IOException e) {
if (i == 0) {
logger.error("Failed to call cluster health - " + e)
}
logger.debug("Call to [{}] threw an exception", url, e)
} finally {
if (httpURLConnection != null) {
httpURLConnection.disconnect();
}
}
// did not start, so wait a bit before trying again
Thread.sleep(750L);
}
return false;
WaitForHttpResource http = new WaitForHttpResource("https", node.httpUri(), numNodes)
http.setTrustStoreFile(caFile)
http.setTrustStorePassword("password")
http.setUsername("test_admin")
http.setPassword("x-pack-test-password")
return http.wait(5000)
}
}
Loading

0 comments on commit c652917

Please sign in to comment.