Skip to content
This repository has been archived by the owner on Mar 31, 2024. It is now read-only.

Commit

Permalink
Add https support to ES exporter and Sense
Browse files Browse the repository at this point in the history
Host setting can now specify a protocol + sub path (i.e., https://host:9200/subpath/ ).
Custom SSL settings can be configured using `marvel.agent.exporter.es.ssl.*` settings. Those will fall back to Shield `shield.ssl` equivalent settings if found.

Sense will now fall back to using https by default if running under https. It will also properly set the withCredentials parameter of XHR to allow for basic auth under CORS setting.

Closes elastic#331
  • Loading branch information
bleskes committed Oct 28, 2014
1 parent a2e20a8 commit d7dab15
Show file tree
Hide file tree
Showing 7 changed files with 317 additions and 62 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,10 @@
import org.elasticsearch.marvel.agent.event.Event;
import org.elasticsearch.node.settings.NodeSettingsService;

import javax.net.ssl.*;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URL;
import java.net.*;
import java.security.KeyStore;
import java.util.ArrayList;
import java.util.Map;
import java.util.regex.Matcher;
Expand All @@ -76,10 +75,15 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
volatile int timeoutInMillis;
volatile int readTimeoutInMillis;


/** https support * */
final SSLSocketFactory sslSocketFactory;

final ClusterService clusterService;
final ClusterName clusterName;
HttpServer httpServer;


public final static DateTimeFormatter defaultDatePrinter = Joda.forPattern("date_time").printer();

volatile boolean checkedAndUploadedIndexTemplate = false;
Expand All @@ -91,19 +95,31 @@ public class ESExporter extends AbstractLifecycleComponent<ESExporter> implement
final ClusterStatsRenderer clusterStatsRenderer;
final EventsRenderer eventsRenderer;

ConnectionKeepAliveWorker keepAliveWorker;
final ConnectionKeepAliveWorker keepAliveWorker;
Thread keepAliveThread;

@Inject
public ESExporter(Settings settings, ClusterService clusterService, ClusterName clusterName,
@ClusterDynamicSettings DynamicSettings dynamicSettings, NodeSettingsService nodeSettingsService) {
@ClusterDynamicSettings DynamicSettings dynamicSettings,
NodeSettingsService nodeSettingsService) {
super(settings);

this.clusterService = clusterService;

this.clusterName = clusterName;

hosts = settings.getAsArray(SETTINGS_HOSTS, Strings.EMPTY_ARRAY);

for (String host : hosts) {
try {
parseHostWithPath(host, "");
} catch (URISyntaxException e) {
throw new RuntimeException("[marvel.agent.exporter] invalid host: [" + host + "]. error: [" + e.getMessage() + "]");
} catch (MalformedURLException e) {
throw new RuntimeException("[marvel.agent.exporter] invalid host: [" + host + "]. error: [" + e.getMessage() + "]");
}
}

indexPrefix = settings.get(SETTINGS_INDEX_PREFIX, ".marvel");
String indexTimeFormat = settings.get(SETTINGS_INDEX_TIME_FORMAT, "YYYY.MM.dd");
indexTimeFormatter = DateTimeFormat.forPattern(indexTimeFormat).withZoneUTC();
Expand All @@ -125,6 +141,14 @@ public ESExporter(Settings settings, ClusterService clusterService, ClusterName
dynamicSettings.addDynamicSetting(SETTINGS_READ_TIMEOUT);
nodeSettingsService.addListener(this);

if (!settings.getByPrefix(SETTINGS_SSL_PREFIX).getAsMap().isEmpty() ||
!settings.getByPrefix(SETTINGS_SSL_SHIELD_PREFIX).getAsMap().isEmpty()) {
sslSocketFactory = createSSLSocketFactory(settings);
} else {
logger.trace("no ssl context configured");
sslSocketFactory = null;
}

logger.debug("initialized with targets: {}, index prefix [{}], index time format [{}]", hosts, indexPrefix, indexTimeFormat);
}

Expand Down Expand Up @@ -282,7 +306,7 @@ protected void doStart() {

@Override
protected void doStop() {
if (keepAliveWorker != null && keepAliveThread.isAlive()) {
if (keepAliveThread != null && keepAliveThread.isAlive()) {
keepAliveWorker.closed = true;
keepAliveThread.interrupt();
try {
Expand Down Expand Up @@ -352,14 +376,14 @@ private HttpURLConnection openConnection(String method, String path, String cont
for (; hostIndex < hosts.length; hostIndex++) {
String host = hosts[hostIndex];
try {
URL url = new URL("http://" + host + "/" + path);
if (url.getPort() == -1) {
// url has no port, default to 9200
host = host + ":9200";
url = new URL("http://" + host + "/" + path);
}
final URL url = parseHostWithPath(host, path);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();

if (conn instanceof HttpsURLConnection && sslSocketFactory != null) {
HttpsURLConnection httpsConn = (HttpsURLConnection) conn;
httpsConn.setSSLSocketFactory(sslSocketFactory);
}

conn.setRequestMethod(method);
conn.setConnectTimeout(timeoutInMillis);
conn.setReadTimeout(readTimeoutInMillis);
Expand All @@ -377,6 +401,8 @@ private HttpURLConnection openConnection(String method, String path, String cont
conn.connect();

return conn;
} catch (URISyntaxException e) {
logger.error("error parsing host [{}]", e, host);
} catch (IOException e) {
logger.error("error connecting to [{}]", e, host);
}
Expand All @@ -397,6 +423,28 @@ private HttpURLConnection openConnection(String method, String path, String cont
return null;
}

static URL parseHostWithPath(String host, String path) throws URISyntaxException, MalformedURLException {

if (!host.contains("://")) {
// prefix with http
host = "http://" + host;
}
if (!host.endsWith("/")) {
// make sure we can safely resolves sub paths and not replace parent folders
host = host + "/";
}

URI hostUrl = new URI(host);

if (hostUrl.getPort() == -1) {
// url has no port, default to 9200
hostUrl = new URI(hostUrl.getScheme(), hostUrl.getUserInfo(), hostUrl.getHost(), 9200, hostUrl.getPath(), hostUrl.getQuery(), hostUrl.getFragment());

}
URI hostWithPath = hostUrl.resolve(path);
return hostWithPath.toURL();
}

static int parseIndexVersionFromTemplate(byte[] template) throws UnsupportedEncodingException {
Pattern versionRegex = Pattern.compile("marvel.index_format\"\\s*:\\s*\"?(\\d+)\"?");
Matcher matcher = versionRegex.matcher(new String(template, "UTF-8"));
Expand Down Expand Up @@ -777,5 +825,72 @@ public void run() {
}
}
}

private static final String SETTINGS_SSL_PREFIX = SETTINGS_PREFIX + "ssl.";
private static final String SETTINGS_SSL_SHIELD_PREFIX = "shield.ssl.";

public static final String SETTINGS_SSL_CONTEXT_ALGORITHM = SETTINGS_SSL_PREFIX + "context_algorithm";
private static final String SETTINGS_SSL_SHIELD_CONTEXT_ALGORITHM = SETTINGS_SSL_SHIELD_PREFIX + "context_algorithm";
public static final String SETTINGS_SSL_TRUSTSTORE = SETTINGS_SSL_PREFIX + "truststore";
private static final String SETTINGS_SSL_SHIELD_TRUSTSTORE = SETTINGS_SSL_SHIELD_PREFIX + "truststore";
public static final String SETTINGS_SSL_TRUSTSTORE_PASSWORD = SETTINGS_SSL_PREFIX + "truststore_password";
private static final String SETTINGS_SSL_SHIELD_TRUSTSTORE_PASSWORD = SETTINGS_SSL_SHIELD_PREFIX + "truststore_password";
public static final String SETTINGS_SSL_TRUSTSTORE_ALGORITHM = SETTINGS_SSL_PREFIX + "truststore_algorithm";
private static final String SETTINGS_SSL_SHIELD_TRUSTSTORE_ALGORITHM = SETTINGS_SSL_SHIELD_PREFIX + "truststore_algorithm";


/** SSL Initialization * */
public SSLSocketFactory createSSLSocketFactory(Settings settings) {
SSLContext sslContext;
// Initialize sslContext
try {
String sslContextAlgorithm = settings.get(SETTINGS_SSL_CONTEXT_ALGORITHM, settings.get(SETTINGS_SSL_SHIELD_CONTEXT_ALGORITHM, "TLS"));
String trustStore = settings.get(SETTINGS_SSL_TRUSTSTORE, settings.get(SETTINGS_SSL_SHIELD_TRUSTSTORE, System.getProperty("javax.net.ssl.trustStore")));
String trustStorePassword = settings.get(SETTINGS_SSL_TRUSTSTORE_PASSWORD, settings.get(SETTINGS_SSL_SHIELD_TRUSTSTORE_PASSWORD, System.getProperty("javax.net.ssl.trustStorePassword")));
String trustStoreAlgorithm = settings.get(SETTINGS_SSL_TRUSTSTORE_ALGORITHM, settings.get(SETTINGS_SSL_SHIELD_TRUSTSTORE_ALGORITHM, System.getProperty("ssl.TrustManagerFactory.algorithm")));

if (trustStore == null) {
throw new RuntimeException("truststore is not configured, use " + SETTINGS_SSL_TRUSTSTORE);
}

if (trustStoreAlgorithm == null) {
trustStoreAlgorithm = TrustManagerFactory.getDefaultAlgorithm();
}

logger.debug("SSL: using trustStore[{}], trustAlgorithm[{}]", trustStore, trustStoreAlgorithm);

if (!new File(trustStore).exists()) {
throw new FileNotFoundException("Truststore at path [" + trustStore + "] does not exist");
}

FileInputStream trustStoreStream = null;
TrustManager[] trustManagers;
try {
trustStoreStream = new FileInputStream(trustStore);
// Load TrustStore
KeyStore ks = KeyStore.getInstance("jks");
ks.load(trustStoreStream, trustStorePassword == null ? null : trustStorePassword.toCharArray());

// Initialize a trust manager factory with the trusted store
TrustManagerFactory trustFactory = TrustManagerFactory.getInstance(trustStoreAlgorithm);
trustFactory.init(ks);

// Retrieve the trust managers from the factory
trustManagers = trustFactory.getTrustManagers();
} catch (Exception e) {
throw new RuntimeException("Failed to initialize a TrustManagerFactory", e);
} finally {
if (trustStoreStream != null) {
trustStoreStream.close();
}
}

sslContext = SSLContext.getInstance(sslContextAlgorithm);
sslContext.init(null, trustManagers, null);
} catch (Exception e) {
throw new RuntimeException("[marvel.agent.exporter] failed to initialize the SSLContext", e);
}
return sslContext.getSocketFactory();
}
}

Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/



package org.elasticsearch.marvel.agent.exporter;

import org.elasticsearch.Version;
Expand All @@ -25,41 +24,22 @@
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.transport.LocalTransportAddress;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.marvel.agent.event.ClusterEvent;
import org.elasticsearch.marvel.agent.event.Event;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.Test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;


// Transport Client instantiation also calls the marvel plugin, which then fails to find modules
@ClusterScope(transportClientRatio = 0.0, scope = ElasticsearchIntegrationTest.Scope.TEST, numNodes = 0)
public class ESExporterTests extends ElasticsearchIntegrationTest {

@Test
public void testVersionIsExtractableFromIndexTemplate() throws IOException {
byte[] template;
InputStream is = ESExporter.class.getResourceAsStream("/marvel_index_template.json");
if (is == null) {
throw new FileNotFoundException("Resource [/marvel_index_template.json] not found in classpath");
}
template = Streams.copyToByteArray(is);
is.close();
MatcherAssert.assertThat(ESExporter.parseIndexVersionFromTemplate(template), Matchers.greaterThan(0));
}

@Test
public void testLargeClusterStateSerialization() throws InterruptedException {
// make sure not other exporting is done (quicker)..
Expand All @@ -83,6 +63,5 @@ public void testLargeClusterStateSerialization() throws InterruptedException {
ensureYellow();
client().admin().indices().prepareRefresh(".marvel-*").get();
assertHitCount(client().prepareSearch().setQuery(QueryBuilders.termQuery("event_source", "test_source_unique")).get(), 1);

}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/**
* ELASTICSEARCH CONFIDENTIAL
* _____________________________
*
* [2014] Elasticsearch Incorporated All Rights Reserved.
*
* NOTICE: All information contained herein is, and remains
* the property of Elasticsearch Incorporated and its suppliers,
* if any. The intellectual and technical concepts contained
* herein are proprietary to Elasticsearch Incorporated
* and its suppliers and may be covered by U.S. and Foreign Patents,
* patents in process, and are protected by trade secret or copyright law.
* Dissemination of this information or reproduction of this material
* is strictly forbidden unless prior written permission is obtained
* from Elasticsearch Incorporated.
*/


package org.elasticsearch.marvel.agent.exporter;

import org.elasticsearch.common.io.Streams;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.Test;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;

import static org.hamcrest.CoreMatchers.equalTo;


public class ESExporterUnitTests extends ElasticsearchTestCase {

@Test
public void testVersionIsExtractableFromIndexTemplate() throws IOException {
byte[] template;
InputStream is = ESExporter.class.getResourceAsStream("/marvel_index_template.json");
if (is == null) {
throw new FileNotFoundException("Resource [/marvel_index_template.json] not found in classpath");
}
template = Streams.copyToByteArray(is);
is.close();
MatcherAssert.assertThat(ESExporter.parseIndexVersionFromTemplate(template), Matchers.greaterThan(0));
}

@Test
public void testHostParsing() throws MalformedURLException, URISyntaxException {
URL url = ESExporter.parseHostWithPath("localhost:9200", "");
verifyUrl(url, "http", "localhost", 9200, "/");

url = ESExporter.parseHostWithPath("localhost", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk");

url = ESExporter.parseHostWithPath("http://localhost:9200", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk");

url = ESExporter.parseHostWithPath("http://localhost", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk");

url = ESExporter.parseHostWithPath("https://localhost:9200", "_bulk");
verifyUrl(url, "https", "localhost", 9200, "/_bulk");

url = ESExporter.parseHostWithPath("https://boaz-air.local:9200", "_bulk");
verifyUrl(url, "https", "boaz-air.local", 9200, "/_bulk");

url = ESExporter.parseHostWithPath("boaz:test@localhost:9200", "");
verifyUrl(url, "http", "localhost", 9200, "/", "boaz:test");

url = ESExporter.parseHostWithPath("boaz:test@localhost", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("http://boaz:test@localhost:9200", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("http://boaz:test@localhost", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("https://boaz:test@localhost:9200", "_bulk");
verifyUrl(url, "https", "localhost", 9200, "/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("boaz:test@localhost:9200/suburl", "");
verifyUrl(url, "http", "localhost", 9200, "/suburl/", "boaz:test");

url = ESExporter.parseHostWithPath("boaz:test@localhost:9200/suburl/", "");
verifyUrl(url, "http", "localhost", 9200, "/suburl/", "boaz:test");

url = ESExporter.parseHostWithPath("localhost/suburl", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/suburl/_bulk");

url = ESExporter.parseHostWithPath("http://boaz:test@localhost:9200/suburl/suburl1", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/suburl/suburl1/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("http://boaz:test@localhost/suburl", "_bulk");
verifyUrl(url, "http", "localhost", 9200, "/suburl/_bulk", "boaz:test");

url = ESExporter.parseHostWithPath("https://boaz:test@localhost:9200/suburl", "_bulk");
verifyUrl(url, "https", "localhost", 9200, "/suburl/_bulk", "boaz:test");
}

void verifyUrl(URL url, String protocol, String host, int port, String path) {
assertThat(url.getProtocol(), equalTo(protocol));
assertThat(url.getHost(), equalTo(host));
assertThat(url.getPort(), equalTo(port));
assertThat(url.getPath(), equalTo(path));
}

void verifyUrl(URL url, String protocol, String host, int port, String path, String userInfo) {
verifyUrl(url, protocol, host, port, path);
assertThat(url.getUserInfo(), equalTo(userInfo));

}
}
Loading

0 comments on commit d7dab15

Please sign in to comment.