From c7e3a7a945893d2c442f5aa8fe125b007cb461e5 Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 13:02:43 +0200 Subject: [PATCH 01/18] Changes bindOnLocalhost=boolean to bindAddress. Introduces advertisedAddress. --- conf/broker.conf | 7 +- conf/standalone.conf | 7 +- conf/websocket.conf | 7 +- .../pulsar/broker/ServiceConfiguration.java | 44 ++++++++-- .../yahoo/pulsar/broker/PulsarService.java | 37 ++++---- .../pulsar/broker/admin/AdminApiTest.java | 2 +- .../auth/MockedPulsarServiceBaseTest.java | 2 +- .../broker/service/AdvertisedAddressTest.java | 84 +++++++++++++++++++ .../service/BacklogQuotaManagerTest.java | 2 +- .../broker/service/ReplicatorTestBase.java | 6 +- .../configurations/pulsar_broker_test.conf | 3 +- 11 files changed, 165 insertions(+), 36 deletions(-) create mode 100644 pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java diff --git a/conf/broker.conf b/conf/broker.conf index 08aa7fcd51b26..f41893aa3c203 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -37,8 +37,11 @@ webServicePortTls=8443 # Enable the WebSocket API service in broker webSocketServiceEnabled=false -# Control whether to bind directly on localhost rather than on normal hostname -bindOnLocalhost=false +# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). +bindAddress= + +# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. +advertisedAddress= # Name of the cluster to which this broker belongs to clusterName= diff --git a/conf/standalone.conf b/conf/standalone.conf index 11a9c16a3fc6f..d65afeaddf804 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -27,8 +27,11 @@ brokerServicePort=6650 # Port to use to server HTTP request webServicePort=8080 -# Control whether to bind directly on localhost rather than on normal hostname -bindOnLocalhost=true +# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). +bindAddress= + +# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. +advertisedAddress= # Name of the cluster to which this broker belongs to clusterName=standalone diff --git a/conf/websocket.conf b/conf/websocket.conf index d0eee0351c7b2..c74e9f7644705 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -22,8 +22,11 @@ globalZookeeperServers= # Port to use to server HTTP request webServicePort=8080 -# Control whether to bind directly on localhost rather than on normal hostname -bindOnLocalhost=false +# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). +bindAddress= + +# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. +advertisedAddress= # Name of the pulsar cluster to connect to clusterName= diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index c60ee3f67b87a..31138c535ba22 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -15,6 +15,8 @@ */ package com.yahoo.pulsar.broker; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Properties; @@ -22,6 +24,8 @@ import com.google.common.collect.Sets; import com.yahoo.pulsar.client.impl.auth.AuthenticationDisabled; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @@ -30,6 +34,8 @@ */ public class ServiceConfiguration { + private static final Logger LOG = LoggerFactory.getLogger(ServiceConfiguration.class); + /***** --- pulsar configuration --- ****/ // Zookeeper quorum connection string @FieldContext(required = true) @@ -43,9 +49,13 @@ public class ServiceConfiguration { private int webServicePort = 8080; // Port to use to server HTTPS request private int webServicePortTls = 8443; - // Control whether to bind directly on localhost rather than on normal - // hostname - private boolean bindOnLocalhost = false; + + // Hostname or IP address the service binds on. + // If not set, InetAddress.getLocalHost().getHostName() will be used. + private String bindAddress; + // Controls which hostname is advertised to the discovery service via ZooKeeper. + // If not set, bindAddress is used. + private String advertisedAddress; // Enable the WebSocket API service private boolean webSocketServiceEnabled = false; @@ -291,12 +301,32 @@ public void setWebServicePortTls(int webServicePortTls) { this.webServicePortTls = webServicePortTls; } - public boolean isBindOnLocalhost() { - return bindOnLocalhost; + public String getBindAddress() { + if (this.bindAddress == null) { + try { + LOG.debug("bindAddress not set, attempting default configuration."); + this.setBindAddress(InetAddress.getLocalHost().getHostName()); + } catch (UnknownHostException ex) { + LOG.warn(ex.getMessage(), "Using localhost as bindAddress."); + this.setAdvertisedAddress("localhost"); + } + } + return this.bindAddress; + } + + public void setBindAddress(String bindAddress) { + this.bindAddress = bindAddress; + } + + public String getAdvertisedAddress() { + if (this.advertisedAddress == null) { + return this.getBindAddress(); + } + return this.advertisedAddress; } - public void setBindOnLocalhost(boolean bindOnLocalhost) { - this.bindOnLocalhost = bindOnLocalhost; + public void setAdvertisedAddress(String advertisedAddress) { + this.advertisedAddress = advertisedAddress; } public boolean isWebSocketServiceEnabled() { diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java index acd915d05109f..e8fe5c099cb80 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java @@ -98,6 +98,7 @@ public class PulsarService implements AutoCloseable { private PulsarAdmin adminClient = null; private ZooKeeperClientFactory zkClientFactory = null; private final String host; + private final String advertisedAddress; private final String webServiceAddress; private final String webServiceAddressTls; private final String brokerServiceUrl; @@ -119,6 +120,7 @@ public enum State { public PulsarService(ServiceConfiguration config) { state = State.Init; this.host = host(config); + this.advertisedAddress = advertisedHost(config); this.webServiceAddress = webAddress(config); this.webServiceAddressTls = webAddressTls(config); this.brokerServiceUrl = brokerUrl(config); @@ -565,41 +567,40 @@ public MessagingServiceShutdownHook getShutdownService() { /** * Derive the host * - * @param isBindOnLocalhost - * @return + * @return Hostname or IP address the service binds on. */ public static String host(ServiceConfiguration config) { - try { - if (!config.isBindOnLocalhost()) { - return InetAddress.getLocalHost().getHostName(); - } else { - return "localhost"; - } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IllegalStateException("failed to find host", e); - } + return config.getBindAddress(); + } + + /** + * Advertised service host. + * + * @return Hostname or IP address the service advertises to the outside world. + */ + public static String advertisedHost(ServiceConfiguration config) { + return config.getAdvertisedAddress(); } public static String brokerUrl(ServiceConfiguration config) { - return "pulsar://" + host(config) + ":" + config.getBrokerServicePort(); + return "pulsar://" + advertisedHost(config) + ":" + config.getBrokerServicePort(); } public static String brokerUrlTls(ServiceConfiguration config) { if (config.isTlsEnabled()) { - return "pulsar://" + host(config) + ":" + config.getBrokerServicePortTls(); + return "pulsar://" + advertisedHost(config) + ":" + config.getBrokerServicePortTls(); } else { return ""; } } public static String webAddress(ServiceConfiguration config) { - return String.format("http://%s:%d", host(config), config.getWebServicePort()); + return String.format("http://%s:%d", advertisedHost(config), config.getWebServicePort()); } public static String webAddressTls(ServiceConfiguration config) { if (config.isTlsEnabled()) { - return String.format("https://%s:%d", host(config), config.getWebServicePortTls()); + return String.format("https://%s:%d", advertisedHost(config), config.getWebServicePortTls()); } else { return ""; } @@ -609,6 +610,10 @@ public String getHost() { return host; } + public String getAdvertisedAddress() { + return advertisedAddress; + } + public String getWebServiceAddress() { return webServiceAddress; } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java index 5deafe4fe7b60..7c1af9d8664c7 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java @@ -120,7 +120,7 @@ public void setup() throws Exception { otherconfig.setBrokerServicePort(SECONDARY_BROKER_PORT); otherconfig.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT); otherconfig.setLoadBalancerEnabled(false); - otherconfig.setBindOnLocalhost(true); + otherconfig.setBindAddress("localhost"); otherconfig.setClusterName("test"); otherPulsar = startBroker(otherconfig); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 13d318b5006a4..15c37fdeaee02 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -74,7 +74,7 @@ public MockedPulsarServiceBaseTest() { this.conf.setBrokerServicePortTls(BROKER_PORT_TLS); this.conf.setWebServicePort(BROKER_WEBSERVICE_PORT); this.conf.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS); - this.conf.setBindOnLocalhost(true); + this.conf.setBindAddress("localhost"); this.conf.setClusterName("test"); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java new file mode 100644 index 0000000000000..8d1eb779dece9 --- /dev/null +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java @@ -0,0 +1,84 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.broker.service; + +import com.yahoo.pulsar.broker.PulsarService; +import com.yahoo.pulsar.broker.ServiceConfiguration; +import com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble; +import org.apache.zookeeper.data.Stat; +import org.json.JSONObject; +import org.json.JSONStringer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +import static com.yahoo.pulsar.broker.ServiceConfigurationLoader.create; + +public class AdvertisedAddressTest { + + LocalBookkeeperEnsemble bkEnsemble; + PulsarService pulsar; + + private final int ZOOKEEPER_PORT = 12759; + private final int BROKER_WEBSERVICE_PORT = 15782; + private final int BROKER_SERVICE_PORT = 16650; + + private final String bindAddress = "127.0.0.1"; + private final String advertisedAddress = "pulsar-usc.example.com"; + + @Before + public void setup() throws Exception { + bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, 5001); + bkEnsemble.start(); + ServiceConfiguration config = create(new Properties(System.getProperties())); + config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); + config.setWebServicePort(BROKER_WEBSERVICE_PORT); + config.setClusterName("usc"); + config.setBrokerServicePort(BROKER_SERVICE_PORT); + config.setBindAddress(bindAddress); + config.setAdvertisedAddress(advertisedAddress); + config.setManagedLedgerMaxEntriesPerLedger(5); + config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); + pulsar = new PulsarService(config); + pulsar.start(); + } + + @After + public void shutdown() throws Exception { + pulsar.close(); + bkEnsemble.stop(); + } + + @Test + public void testAdvertisedAddress() throws Exception { + Assert.assertEquals( pulsar.getHost(), bindAddress ); + Assert.assertEquals( pulsar.getAdvertisedAddress(), advertisedAddress ); + Assert.assertEquals( pulsar.getBrokerServiceUrl(), String.format("pulsar://%s:%d", advertisedAddress, BROKER_SERVICE_PORT) ); + Assert.assertEquals( pulsar.getWebServiceAddress(), String.format("http://%s:%d", advertisedAddress, BROKER_WEBSERVICE_PORT) ); + String brokerZkPath = String.format("/loadbalance/brokers/%s:%d", bindAddress, BROKER_WEBSERVICE_PORT); + String bkBrokerData = new String(bkEnsemble.getZkClient().getData(brokerZkPath, false, new Stat()), StandardCharsets.UTF_8); + JSONObject jsonBkBrokerData = new JSONObject(bkBrokerData); + Assert.assertEquals( jsonBkBrokerData.get("pulsarServiceUrl"), pulsar.getBrokerServiceUrl() ); + Assert.assertEquals( jsonBkBrokerData.get("webServiceUrl"), pulsar.getWebServiceAddress() ); + } + +} diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java index 03e266a959e3b..903717a6276e3 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -84,7 +84,7 @@ void setup() throws Exception { config.setBrokerServicePort(BROKER_SERVICE_PORT); config.setAuthorizationEnabled(false); config.setAuthenticationEnabled(false); - config.setBindOnLocalhost(true); + config.setBindAddress("localhost"); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java index 84304897c733a..caa17cad218ea 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java @@ -113,7 +113,7 @@ void setup() throws Exception { config1.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config1.setBrokerServicePort(PortManager.nextFreePort()); - config1.setBindOnLocalhost(true); + config1.setBindAddress("localhost"); pulsar1 = new PulsarService(config1); pulsar1.start(); ns1 = pulsar1.getBrokerService(); @@ -138,7 +138,7 @@ void setup() throws Exception { config2.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config2.setBrokerServicePort(PortManager.nextFreePort()); - config2.setBindOnLocalhost(true); + config2.setBindAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); ns2 = pulsar2.getBrokerService(); @@ -163,7 +163,7 @@ void setup() throws Exception { config3.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config3.setBrokerServicePort(PortManager.nextFreePort()); - config3.setBindOnLocalhost(true); + config3.setBindAddress("localhost"); pulsar3 = new PulsarService(config3); pulsar3.start(); ns3 = pulsar3.getBrokerService(); diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf index fbc6c396776ff..a6c2e917f766d 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf @@ -5,7 +5,8 @@ brokerServicePort=6650 brokerServicePortTls=6651 webServicePort=8080 webServicePortTls=4443 -bindOnLocalhost=false +bindAddress= +advertisedAddress= clusterName="test_cluster" brokerShutdownTimeoutMs=3000 backlogQuotaCheckEnabled=true From cc08ed3fde5292a9ea685ae5975028de405d5f77 Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 19:57:14 +0200 Subject: [PATCH 02/18] Removes advertisement from web socket config. --- conf/websocket.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/conf/websocket.conf b/conf/websocket.conf index c74e9f7644705..c30d461f05869 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -25,9 +25,6 @@ webServicePort=8080 # Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). bindAddress= -# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. -advertisedAddress= - # Name of the pulsar cluster to connect to clusterName= From 6fa126f47fe11d3a34752013f7bba41c8ed882ae Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 23:35:09 +0200 Subject: [PATCH 03/18] ENsure services bind on the bind address. --- .../java/com/yahoo/pulsar/broker/service/BrokerService.java | 5 +++-- .../main/java/com/yahoo/pulsar/broker/web/WebService.java | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java index 263337ff47ea4..4ed0a0654f100 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java @@ -22,6 +22,7 @@ import java.io.Closeable; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.util.HashMap; import java.util.List; @@ -220,13 +221,13 @@ public void start() throws Exception { bootstrap.childHandler(new PulsarChannelInitializer(this, serviceConfig, false)); // Bind and start to accept incoming connections. - bootstrap.bind(port).sync(); + bootstrap.bind(new InetSocketAddress(pulsar.getHost(), port)).sync(); log.info("Started Pulsar Broker service on port {}", port); if (serviceConfig.isTlsEnabled()) { ServerBootstrap tlsBootstrap = bootstrap.clone(); tlsBootstrap.childHandler(new PulsarChannelInitializer(this, serviceConfig, true)); - tlsBootstrap.bind(tlsPort).sync(); + tlsBootstrap.bind(new InetSocketAddress(pulsar.getHost(), tlsPort)).sync(); log.info("Started Pulsar Broker TLS service on port {}", tlsPort); } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java index 1877a7d5851a5..f86716dccd5c9 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java @@ -85,6 +85,7 @@ public WebService(ServiceConfiguration config, PulsarService pulsar) throws Puls ServerConnector connector = new PulsarServerConnector(server, 1, 1); connector.setPort(config.getWebServicePort()); + connector.setHost(pulsar.getHost()); connectors.add(connector); if (config.isTlsEnabled()) { @@ -102,6 +103,7 @@ public WebService(ServiceConfiguration config, PulsarService pulsar) throws Puls sslCtxFactory.setWantClientAuth(true); ServerConnector tlsConnector = new PulsarServerConnector(server, 1, 1, sslCtxFactory); tlsConnector.setPort(config.getWebServicePortTls()); + tlsConnector.setHost(pulsar.getHost()); connectors.add(tlsConnector); } From 37450339cc00d11793ebef16b9a3959526bc8825 Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 23:37:46 +0200 Subject: [PATCH 04/18] Change bind address and advertised address hostname resolution to as suggested by @merlimat. --- .../pulsar/broker/ServiceConfiguration.java | 12 ------------ .../com/yahoo/pulsar/broker/PulsarService.java | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index 31138c535ba22..303531eee04f9 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -302,15 +302,6 @@ public void setWebServicePortTls(int webServicePortTls) { } public String getBindAddress() { - if (this.bindAddress == null) { - try { - LOG.debug("bindAddress not set, attempting default configuration."); - this.setBindAddress(InetAddress.getLocalHost().getHostName()); - } catch (UnknownHostException ex) { - LOG.warn(ex.getMessage(), "Using localhost as bindAddress."); - this.setAdvertisedAddress("localhost"); - } - } return this.bindAddress; } @@ -319,9 +310,6 @@ public void setBindAddress(String bindAddress) { } public String getAdvertisedAddress() { - if (this.advertisedAddress == null) { - return this.getBindAddress(); - } return this.advertisedAddress; } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java index e8fe5c099cb80..9542800e604f8 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.URL; +import java.net.UnknownHostException; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -570,6 +571,14 @@ public MessagingServiceShutdownHook getShutdownService() { * @return Hostname or IP address the service binds on. */ public static String host(ServiceConfiguration config) { + if (config.getBindAddress() == null) { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException ex) { + LOG.error(ex.getMessage(), ex); + throw new IllegalStateException("Failed to resolve localhost name.", ex); + } + } return config.getBindAddress(); } @@ -579,6 +588,14 @@ public static String host(ServiceConfiguration config) { * @return Hostname or IP address the service advertises to the outside world. */ public static String advertisedHost(ServiceConfiguration config) { + if (config.getAdvertisedAddress() == null) { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException ex) { + LOG.error(ex.getMessage(), ex); + throw new IllegalStateException("Failed to resolve localhost name.", ex); + } + } return config.getAdvertisedAddress(); } From 393e10ee24e82c08fbe590dba5c05a35dc8f692a Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 23:43:15 +0200 Subject: [PATCH 05/18] Remove unnecessary LOG in ServiceConfiguration. --- .../java/com/yahoo/pulsar/broker/ServiceConfiguration.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index 303531eee04f9..82f597db7a018 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -24,8 +24,6 @@ import com.google.common.collect.Sets; import com.yahoo.pulsar.client.impl.auth.AuthenticationDisabled; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @@ -34,8 +32,6 @@ */ public class ServiceConfiguration { - private static final Logger LOG = LoggerFactory.getLogger(ServiceConfiguration.class); - /***** --- pulsar configuration --- ****/ // Zookeeper quorum connection string @FieldContext(required = true) From 3ed746e20e5a4a242c7d59d66945eb89a2c475db Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 23:52:08 +0200 Subject: [PATCH 06/18] Removed unused imports. --- .../main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index 82f597db7a018..e857e2b0fe957 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -15,8 +15,6 @@ */ package com.yahoo.pulsar.broker; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; import java.util.Properties; From 1bec010263aeaa1388b79f8dcac1b4a1ec6fec4a Mon Sep 17 00:00:00 2001 From: radekg Date: Tue, 20 Sep 2016 23:58:13 +0200 Subject: [PATCH 07/18] Corrections in comments. --- conf/standalone.conf | 2 +- .../java/com/yahoo/pulsar/broker/ServiceConfiguration.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conf/standalone.conf b/conf/standalone.conf index d65afeaddf804..6f68da6087f7a 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -30,7 +30,7 @@ webServicePort=8080 # Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). bindAddress= -# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. +# Hostname or IP address the service advertises to the outside world, default is InetAddress.getLocalHost().getHostName(). advertisedAddress= # Name of the cluster to which this broker belongs to diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index e857e2b0fe957..cbc81b6829fb1 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -45,10 +45,9 @@ public class ServiceConfiguration { private int webServicePortTls = 8443; // Hostname or IP address the service binds on. - // If not set, InetAddress.getLocalHost().getHostName() will be used. private String bindAddress; + // Controls which hostname is advertised to the discovery service via ZooKeeper. - // If not set, bindAddress is used. private String advertisedAddress; // Enable the WebSocket API service From bf272606a7b80c3bf0e120b07a6c2689fb44e578 Mon Sep 17 00:00:00 2001 From: radekg Date: Wed, 21 Sep 2016 00:48:41 +0200 Subject: [PATCH 08/18] ServiceConfigurration in tests not loaded from Properties. --- .../pulsar/broker/service/AdvertisedAddressTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java index 8d1eb779dece9..d3649d473f0db 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java @@ -20,18 +20,12 @@ import com.yahoo.pulsar.zookeeper.LocalBookkeeperEnsemble; import org.apache.zookeeper.data.Stat; import org.json.JSONObject; -import org.json.JSONStringer; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; import java.nio.charset.StandardCharsets; -import java.util.Properties; - -import static com.yahoo.pulsar.broker.ServiceConfigurationLoader.create; public class AdvertisedAddressTest { @@ -49,7 +43,7 @@ public class AdvertisedAddressTest { public void setup() throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, 5001); bkEnsemble.start(); - ServiceConfiguration config = create(new Properties(System.getProperties())); + ServiceConfiguration config = new ServiceConfiguration(); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setWebServicePort(BROKER_WEBSERVICE_PORT); config.setClusterName("usc"); From 10ccdb6d1f41148c8aa1d42b68823718fb9d2fc0 Mon Sep 17 00:00:00 2001 From: radekg Date: Fri, 23 Sep 2016 14:29:16 +0200 Subject: [PATCH 09/18] Fixes the hanging tests. --- .../test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java | 1 + .../yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java | 1 + .../com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java | 1 + .../yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java | 1 + .../com/yahoo/pulsar/broker/service/ReplicatorTestBase.java | 3 +++ 5 files changed, 7 insertions(+) diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java index 7c1af9d8664c7..6006f765e3b4f 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java @@ -121,6 +121,7 @@ public void setup() throws Exception { otherconfig.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT); otherconfig.setLoadBalancerEnabled(false); otherconfig.setBindAddress("localhost"); + otherconfig.setAdvertisedAddress("localhost"); otherconfig.setClusterName("test"); otherPulsar = startBroker(otherconfig); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 15c37fdeaee02..5858e9c5d43f0 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -75,6 +75,7 @@ public MockedPulsarServiceBaseTest() { this.conf.setWebServicePort(BROKER_WEBSERVICE_PORT); this.conf.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS); this.conf.setBindAddress("localhost"); + this.conf.setAdvertisedAddress("localhost"); this.conf.setClusterName("test"); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java index d3649d473f0db..28beecd9c7774 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java @@ -49,6 +49,7 @@ public void setup() throws Exception { config.setClusterName("usc"); config.setBrokerServicePort(BROKER_SERVICE_PORT); config.setBindAddress(bindAddress); + config.setAdvertisedAddress(bindAddress); config.setAdvertisedAddress(advertisedAddress); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java index 903717a6276e3..7c1b40cb132a5 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -85,6 +85,7 @@ void setup() throws Exception { config.setAuthorizationEnabled(false); config.setAuthenticationEnabled(false); config.setBindAddress("localhost"); + config.setAdvertisedAddress("localhost"); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java index caa17cad218ea..add95d956c044 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java @@ -114,6 +114,7 @@ void setup() throws Exception { inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config1.setBrokerServicePort(PortManager.nextFreePort()); config1.setBindAddress("localhost"); + config1.setAdvertisedAddress("localhost"); pulsar1 = new PulsarService(config1); pulsar1.start(); ns1 = pulsar1.getBrokerService(); @@ -139,6 +140,7 @@ void setup() throws Exception { inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config2.setBrokerServicePort(PortManager.nextFreePort()); config2.setBindAddress("localhost"); + config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); ns2 = pulsar2.getBrokerService(); @@ -164,6 +166,7 @@ void setup() throws Exception { inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config3.setBrokerServicePort(PortManager.nextFreePort()); config3.setBindAddress("localhost"); + config3.setAdvertisedAddress("localhost"); pulsar3 = new PulsarService(config3); pulsar3.start(); ns3 = pulsar3.getBrokerService(); From 98f01cd0705cbcfb9810e05d56ea2e302cae6cc8 Mon Sep 17 00:00:00 2001 From: radekg Date: Fri, 23 Sep 2016 22:25:20 +0200 Subject: [PATCH 10/18] Fixes to make the tests pass. --- .../src/main/java/com/yahoo/pulsar/broker/PulsarService.java | 3 +-- .../test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java | 4 ++-- .../test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java index ced6373f4cb21..90fd9e857f619 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java @@ -543,8 +543,7 @@ public BookKeeperClientFactory getBookKeeperClientFactory() { public synchronized PulsarAdmin getAdminClient() throws PulsarServerException { if (this.adminClient == null) { try { - String adminApiUrl = "http://" + InetAddress.getLocalHost().getHostName() + ":" - + this.getConfiguration().getWebServicePort(); + String adminApiUrl = webAddress(config); this.adminClient = new PulsarAdmin(new URL(adminApiUrl), this.getConfiguration().getBrokerClientAuthenticationPlugin(), this.getConfiguration().getBrokerClientAuthenticationParameters()); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java index 34257e9dfe56a..16214e50d5b32 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java @@ -67,7 +67,6 @@ public class SLAMonitoringTest { private int[] brokerWebServicePorts = new int[BROKER_COUNT]; private int[] brokerNativeBrokerPorts = new int[BROKER_COUNT]; private URL[] brokerUrls = new URL[BROKER_COUNT]; - private String[] lookupAddresses = new String[BROKER_COUNT]; private PulsarService[] pulsarServices = new PulsarService[BROKER_COUNT]; private PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT]; private ServiceConfiguration[] configurations = new ServiceConfiguration[BROKER_COUNT]; @@ -90,13 +89,14 @@ void setup() throws Exception { config.setWebServicePort(brokerWebServicePorts[i]); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setBrokerServicePort(brokerNativeBrokerPorts[i]); + config.setBindAddress("localhost"); + config.setAdvertisedAddress("localhost"); configurations[i] = config; pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]); - lookupAddresses[i] = pulsarServices[i].getHost() + ":" + config.getWebServicePort(); pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java index 7552109e47a08..06fc9e23a25d4 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java @@ -288,6 +288,8 @@ private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowU config.setTlsAllowInsecureConnection(allowInsecure); config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH); config.setClusterName("local"); + config.setBindAddress("localhost"); + config.setAdvertisedAddress("localhost"); pulsar = spy(new PulsarService(config)); doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory(); pulsar.start(); From 94dde8e3841f16b563371d366d4f9e935959abe5 Mon Sep 17 00:00:00 2001 From: radekg Date: Sat, 24 Sep 2016 01:48:59 +0200 Subject: [PATCH 11/18] Removing the setBindAddress and setAdvertisedAddress from tests. Unit tests failing. --- .gitignore | 2 ++ .../pulsar/broker/ServiceConfiguration.java | 2 +- .../loadbalance/impl/SimpleLoadManagerImpl.java | 6 +++--- .../broker/namespace/NamespaceService.java | 6 +++--- .../yahoo/pulsar/broker/SLAMonitoringTest.java | 16 +++++++--------- .../yahoo/pulsar/broker/admin/AdminApiTest.java | 4 +--- .../com/yahoo/pulsar/broker/admin/AdminTest.java | 2 +- .../broker/auth/MockedPulsarServiceBaseTest.java | 6 ++---- .../broker/loadbalance/LoadBalancerTest.java | 2 +- .../loadbalance/SimpleLoadManagerImplTest.java | 4 ++-- .../broker/namespace/OwnershipCacheTest.java | 2 +- .../broker/service/AdvertisedAddressTest.java | 6 +----- .../broker/service/BacklogQuotaManagerTest.java | 2 -- .../broker/service/ReplicatorTestBase.java | 6 ------ .../yahoo/pulsar/broker/web/WebServiceTest.java | 8 ++++---- .../com/yahoo/pulsar/utils/AddressUtils.java | 16 ++++++++++++++++ .../configurations/pulsar_broker_test.conf | 2 +- 17 files changed, 46 insertions(+), 46 deletions(-) create mode 100644 pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java diff --git a/.gitignore b/.gitignore index 480322603cbb8..f3fd3531cf321 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ test-results dependency-reduced-pom.xml logs /data +pulsar-broker/tmp.* +pulsar-broker/src/test/resources/log4j.properties *.versionsBackup diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java index 2c5992326afea..ec67449827ffe 100644 --- a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfiguration.java @@ -45,7 +45,7 @@ public class ServiceConfiguration { private int webServicePortTls = 8443; // Hostname or IP address the service binds on. - private String bindAddress; + private String bindAddress = "0.0.0.0"; // Controls which hostname is advertised to the discovery service via ZooKeeper. private String advertisedAddress; diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java index 50842ded5d516..a9a1786d84a9a 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/loadbalance/impl/SimpleLoadManagerImpl.java @@ -241,7 +241,7 @@ public void start() throws PulsarServerException { } } - String lookupServiceAddress = pulsar.getHost() + ":" + conf.getWebServicePort(); + String lookupServiceAddress = pulsar.getAdvertisedAddress() + ":" + conf.getWebServicePort(); brokerZnodePath = LOADBALANCE_BROKERS_ROOT + "/" + lookupServiceAddress; LoadReport loadReport = null; try { @@ -637,7 +637,7 @@ public void writeResourceQuotasToZooKeeper() throws Exception { */ private synchronized void doLoadRanking() { ResourceUnitRanking.setCpuUsageByMsgRate(this.realtimeCpuLoadFactor); - String hostname = pulsar.getHost(); + String hostname = pulsar.getAdvertisedAddress(); String strategy = this.getLoadBalancerPlacementStrategy(); log.info("doLoadRanking - load balancing strategy: {}", strategy); if (!currentLoadReports.isEmpty()) { @@ -1094,7 +1094,7 @@ public LoadReport generateLoadReport() throws Exception { try { LoadReport loadReport = new LoadReport(pulsar.getWebServiceAddress(), pulsar.getWebServiceAddressTls(), pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()); - loadReport.setName(String.format("%s:%s", pulsar.getHost(), pulsar.getConfiguration().getWebServicePort())); + loadReport.setName(String.format("%s:%s", pulsar.getAdvertisedAddress(), pulsar.getConfiguration().getWebServicePort())); SystemResourceUsage systemResourceUsage = this.getSystemResourceUsage(); loadReport.setOverLoaded( isAboveLoadLevel(systemResourceUsage, this.getLoadBalancerBrokerOverloadedThresholdPercentage())); diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java index ef6b5ec143cec..bfd946552e0d8 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/namespace/NamespaceService.java @@ -118,7 +118,7 @@ public enum AddressType { */ public NamespaceService(PulsarService pulsar) { this.pulsar = pulsar; - host = pulsar.getHost(); + host = pulsar.getAdvertisedAddress(); this.config = pulsar.getConfiguration(); this.loadManager = pulsar.getLoadManager(); ServiceUnitZkUtils.initZK(pulsar.getLocalZkCache().getZooKeeper(), pulsar.getBrokerServiceUrl()); @@ -446,9 +446,9 @@ private NamespaceOwnershipStatus getNamespaceOwnershipStatus(OwnedServiceUnit ns } // found corresponding policy, set the status to controlled nsOwnedStatus.is_controlled = true; - if (nsIsolationPolicy.isPrimaryBroker(pulsar.getHost())) { + if (nsIsolationPolicy.isPrimaryBroker(pulsar.getAdvertisedAddress())) { nsOwnedStatus.broker_assignment = BrokerAssignment.primary; - } else if (nsIsolationPolicy.isSecondaryBroker(pulsar.getHost())) { + } else if (nsIsolationPolicy.isSecondaryBroker(pulsar.getAdvertisedAddress())) { nsOwnedStatus.broker_assignment = BrokerAssignment.secondary; } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java index 16214e50d5b32..02eb42826b6a6 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/SLAMonitoringTest.java @@ -89,8 +89,6 @@ void setup() throws Exception { config.setWebServicePort(brokerWebServicePorts[i]); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setBrokerServicePort(brokerNativeBrokerPorts[i]); - config.setBindAddress("localhost"); - config.setAdvertisedAddress("localhost"); configurations[i] = config; pulsarServices[i] = new PulsarService(config); @@ -105,7 +103,7 @@ void setup() throws Exception { createProperty(pulsarAdmins[BROKER_COUNT - 1]); for (int i = 0; i < BROKER_COUNT; i++) { String destination = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", - pulsarServices[i].getHost(), brokerWebServicePorts[i]); + pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]); pulsarAdmins[0].namespaces().createNamespace(destination); } } @@ -175,13 +173,13 @@ public void testOwnershipViaAdminAfterSetup() { for (int i = 0; i < BROKER_COUNT; i++) { try { String destination = String.format("persistent://%s/%s/%s:%s/%s", - NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getHost(), + NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i], "my-topic"); assertEquals(pulsarAdmins[0].lookups().lookupDestination(destination), - "pulsar://" + pulsarServices[i].getHost() + ":" + brokerNativeBrokerPorts[i]); + "pulsar://" + pulsarServices[i].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[i]); } catch (Exception e) { e.printStackTrace(); - fail("SLA Namespace should have been owned by the broker(" + "pulsar://" + pulsarServices[i].getHost() + fail("SLA Namespace should have been owned by the broker(" + "pulsar://" + pulsarServices[i].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[i] + ")"); } } @@ -200,7 +198,7 @@ public void testUnloadIfBrokerCrashes() { } String destination = String.format("persistent://%s/%s/%s:%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY, - "my-cluster", pulsarServices[crashIndex].getHost(), brokerWebServicePorts[crashIndex], "my-topic"); + "my-cluster", pulsarServices[crashIndex].getAdvertisedAddress(), brokerWebServicePorts[crashIndex], "my-topic"); log.info("Lookup for namespace {}", destination); @@ -209,7 +207,7 @@ public void testUnloadIfBrokerCrashes() { broker = pulsarAdmins[BROKER_COUNT - 1].lookups().lookupDestination(destination); log.info("{} Namespace is owned by {}", destination, broker); assertNotEquals(broker, - "pulsar://" + pulsarServices[crashIndex].getHost() + ":" + brokerNativeBrokerPorts[crashIndex]); + "pulsar://" + pulsarServices[crashIndex].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[crashIndex]); } catch (PulsarAdminException e) { e.printStackTrace(); fail("The SLA Monitor namespace should be owned by some other broker"); @@ -230,7 +228,7 @@ public void testUnloadIfBrokerCrashes() { broker = pulsarAdmins[0].lookups().lookupDestination(destination); log.info("{} Namespace is re-owned by {}", destination, broker); assertEquals(broker, - "pulsar://" + pulsarServices[crashIndex].getHost() + ":" + brokerNativeBrokerPorts[crashIndex]); + "pulsar://" + pulsarServices[crashIndex].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[crashIndex]); } catch (PulsarAdminException e) { e.printStackTrace(); fail("The SLA Monitor namespace should be reowned by the broker" + broker); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java index 6006f765e3b4f..0d3813f6ccf9d 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminApiTest.java @@ -120,8 +120,6 @@ public void setup() throws Exception { otherconfig.setBrokerServicePort(SECONDARY_BROKER_PORT); otherconfig.setWebServicePort(SECONDARY_BROKER_WEBSERVICE_PORT); otherconfig.setLoadBalancerEnabled(false); - otherconfig.setBindAddress("localhost"); - otherconfig.setAdvertisedAddress("localhost"); otherconfig.setClusterName("test"); otherPulsar = startBroker(otherconfig); @@ -354,7 +352,7 @@ public void brokers() throws Exception { Assert.assertEquals(1, nsMap.size()); for (String ns : nsMap.keySet()) { NamespaceOwnershipStatus nsStatus = nsMap.get(ns); - if (ns.equals(NamespaceService.getHeartbeatNamespace(pulsar.getHost(), pulsar.getConfiguration()) + if (ns.equals(NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration()) + "/0x00000000_0xffffffff")) { assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared); assertFalse(nsStatus.is_controlled); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java index b376db79b92a5..f0e3fb2544e57 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/admin/AdminTest.java @@ -500,7 +500,7 @@ void brokers() throws Exception { Set activeBrokers = brokers.getActiveBrokers("use"); assertEquals(activeBrokers.size(), 1); - assertEquals(activeBrokers, Sets.newHashSet("localhost:" + BROKER_WEBSERVICE_PORT)); + assertEquals(activeBrokers, Sets.newHashSet(pulsar.getAdvertisedAddress() + ":" + BROKER_WEBSERVICE_PORT)); } @Test diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 405d2b960a12b..ad636268d4be6 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -76,8 +76,6 @@ public MockedPulsarServiceBaseTest() { this.conf.setBrokerServicePortTls(BROKER_PORT_TLS); this.conf.setWebServicePort(BROKER_WEBSERVICE_PORT); this.conf.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS); - this.conf.setBindAddress("localhost"); - this.conf.setAdvertisedAddress("localhost"); this.conf.setClusterName("test"); } @@ -103,8 +101,8 @@ private final void init() throws Exception { startBroker(); - brokerUrl = new URL("http://localhost:" + BROKER_WEBSERVICE_PORT); - brokerUrlTls = new URL("https://localhost:" + BROKER_WEBSERVICE_PORT_TLS); + brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + BROKER_WEBSERVICE_PORT); + brokerUrlTls = new URL("https://" + pulsar.getAdvertisedAddress() + ":" + BROKER_WEBSERVICE_PORT_TLS); admin = spy(new PulsarAdmin(brokerUrl, (Authentication) null)); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java index 2ab40ecf181b3..c0e173f070ae4 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/LoadBalancerTest.java @@ -140,7 +140,7 @@ void setup() throws Exception { pulsarServices[i].start(); brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]); - lookupAddresses[i] = pulsarServices[i].getHost() + ":" + config.getWebServicePort(); + lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + config.getWebServicePort(); pulsarAdmins[i] = new PulsarAdmin(brokerUrls[i], (Authentication) null); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java index ca8dc358f7f22..f0d7144e97f3e 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/loadbalance/SimpleLoadManagerImplTest.java @@ -174,7 +174,7 @@ private void createNamespacePolicies(PulsarService pulsar) throws Exception { policyData.namespaces = new ArrayList(); policyData.namespaces.add("pulsar/use/primary-ns.*"); policyData.primary = new ArrayList(); - policyData.primary.add(pulsar1.getHost() + "*"); + policyData.primary.add(pulsar1.getAdvertisedAddress() + "*"); policyData.secondary = new ArrayList(); policyData.secondary.add("prod2-broker([78]).messaging.usw.example.co.*"); policyData.auto_failover_policy = new AutoFailoverPolicyData(); @@ -237,7 +237,7 @@ public void testPrimary() throws Exception { rd.put("bandwidthOut", new ResourceUsage(550 * 1024, 1024 * 1024)); ResourceUnit ru1 = new SimpleResourceUnit( - "http://" + pulsar1.getHost() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd); + "http://" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort(), rd); Set rus = new HashSet(); rus.add(ru1); LoadRanker lr = new ResourceAvailabilityRanker(); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java index 85e119547253b..8afb27c6f0616 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java @@ -80,7 +80,7 @@ public void setup() throws Exception { doReturn(nsService).when(pulsar).getNamespaceService(); doReturn(port).when(config).getBrokerServicePort(); doReturn(brokerService).when(pulsar).getBrokerService(); - doReturn(host(config)).when(pulsar).getHost(); + doReturn(host(config)).when(pulsar).getAdvertisedAddress(); doReturn(webAddress(config)).when(pulsar).getWebServiceAddress(); doReturn(selfBrokerUrl).when(pulsar).getBrokerServiceUrl(); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java index 28beecd9c7774..014304157fd6b 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/AdvertisedAddressTest.java @@ -36,7 +36,6 @@ public class AdvertisedAddressTest { private final int BROKER_WEBSERVICE_PORT = 15782; private final int BROKER_SERVICE_PORT = 16650; - private final String bindAddress = "127.0.0.1"; private final String advertisedAddress = "pulsar-usc.example.com"; @Before @@ -48,8 +47,6 @@ public void setup() throws Exception { config.setWebServicePort(BROKER_WEBSERVICE_PORT); config.setClusterName("usc"); config.setBrokerServicePort(BROKER_SERVICE_PORT); - config.setBindAddress(bindAddress); - config.setAdvertisedAddress(bindAddress); config.setAdvertisedAddress(advertisedAddress); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); @@ -65,11 +62,10 @@ public void shutdown() throws Exception { @Test public void testAdvertisedAddress() throws Exception { - Assert.assertEquals( pulsar.getHost(), bindAddress ); Assert.assertEquals( pulsar.getAdvertisedAddress(), advertisedAddress ); Assert.assertEquals( pulsar.getBrokerServiceUrl(), String.format("pulsar://%s:%d", advertisedAddress, BROKER_SERVICE_PORT) ); Assert.assertEquals( pulsar.getWebServiceAddress(), String.format("http://%s:%d", advertisedAddress, BROKER_WEBSERVICE_PORT) ); - String brokerZkPath = String.format("/loadbalance/brokers/%s:%d", bindAddress, BROKER_WEBSERVICE_PORT); + String brokerZkPath = String.format("/loadbalance/brokers/%s:%d", pulsar.getAdvertisedAddress(), BROKER_WEBSERVICE_PORT); String bkBrokerData = new String(bkEnsemble.getZkClient().getData(brokerZkPath, false, new Stat()), StandardCharsets.UTF_8); JSONObject jsonBkBrokerData = new JSONObject(bkBrokerData); Assert.assertEquals( jsonBkBrokerData.get("pulsarServiceUrl"), pulsar.getBrokerServiceUrl() ); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java index 7c1b40cb132a5..bd7e7199d3c09 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -84,8 +84,6 @@ void setup() throws Exception { config.setBrokerServicePort(BROKER_SERVICE_PORT); config.setAuthorizationEnabled(false); config.setAuthenticationEnabled(false); - config.setBindAddress("localhost"); - config.setAdvertisedAddress("localhost"); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java index add95d956c044..2c39416d9c18b 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/ReplicatorTestBase.java @@ -113,8 +113,6 @@ void setup() throws Exception { config1.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config1.setBrokerServicePort(PortManager.nextFreePort()); - config1.setBindAddress("localhost"); - config1.setAdvertisedAddress("localhost"); pulsar1 = new PulsarService(config1); pulsar1.start(); ns1 = pulsar1.getBrokerService(); @@ -139,8 +137,6 @@ void setup() throws Exception { config2.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config2.setBrokerServicePort(PortManager.nextFreePort()); - config2.setBindAddress("localhost"); - config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); ns2 = pulsar2.getBrokerService(); @@ -165,8 +161,6 @@ void setup() throws Exception { config3.setBrokerServicePurgeInactiveFrequencyInSeconds( inSec(getBrokerServicePurgeInactiveFrequency(), TimeUnit.SECONDS)); config3.setBrokerServicePort(PortManager.nextFreePort()); - config3.setBindAddress("localhost"); - config3.setAdvertisedAddress("localhost"); pulsar3 = new PulsarService(config3); pulsar3.start(); ns3 = pulsar3.getBrokerService(); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java index 06fc9e23a25d4..2963f09b5c8ea 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.InetAddress; import java.net.URL; import java.security.KeyStore; import java.security.PrivateKey; @@ -37,6 +38,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; +import com.yahoo.pulsar.utils.AddressUtils; import org.apache.bookkeeper.test.PortManager; import org.apache.zookeeper.CreateMode; import org.testng.Assert; @@ -68,8 +70,8 @@ public class WebServiceTest { private PulsarService pulsar; private final static int BROKER_WEBSERVICE_PORT = PortManager.nextFreePort(); private final static int BROKER_WEBSERVICE_PORT_TLS = PortManager.nextFreePort(); - private static final String BROKER_URL_BASE = "http://localhost" + ":" + BROKER_WEBSERVICE_PORT; - private static final String BROKER_URL_BASE_TLS = "https://localhost" + ":" + BROKER_WEBSERVICE_PORT_TLS; + private static final String BROKER_URL_BASE = "http://" + AddressUtils.unsafeLocalHostName() + ":" + BROKER_WEBSERVICE_PORT; + private static final String BROKER_URL_BASE_TLS = "https://" + AddressUtils.unsafeLocalHostName() + ":" + BROKER_WEBSERVICE_PORT_TLS; private static final String BROKER_LOOKUP_URL = BROKER_URL_BASE + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic"; private static final String BROKER_LOOKUP_URL_TLS = BROKER_URL_BASE_TLS @@ -288,8 +290,6 @@ private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowU config.setTlsAllowInsecureConnection(allowInsecure); config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH); config.setClusterName("local"); - config.setBindAddress("localhost"); - config.setAdvertisedAddress("localhost"); pulsar = spy(new PulsarService(config)); doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory(); pulsar.start(); diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java new file mode 100644 index 0000000000000..5147743f9d25d --- /dev/null +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java @@ -0,0 +1,16 @@ +package com.yahoo.pulsar.utils; + +import java.net.Inet4Address; +import java.net.UnknownHostException; + +public class AddressUtils { + + public static String unsafeLocalHostName() { + try { + return Inet4Address.getLocalHost().getHostName(); + } catch (UnknownHostException ex) { + throw new RuntimeException(ex); + } + } + +} diff --git a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf index a6c2e917f766d..e31a429e6a6cb 100644 --- a/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf +++ b/pulsar-broker/src/test/resources/configurations/pulsar_broker_test.conf @@ -5,7 +5,7 @@ brokerServicePort=6650 brokerServicePortTls=6651 webServicePort=8080 webServicePortTls=4443 -bindAddress= +bindAddress=0.0.0.0 advertisedAddress= clusterName="test_cluster" brokerShutdownTimeoutMs=3000 From 2b9c25cedaf8d6f918a8cd139d5e9d39896220e0 Mon Sep 17 00:00:00 2001 From: radekg Date: Sat, 24 Sep 2016 02:31:56 +0200 Subject: [PATCH 12/18] Fixed TLS tests. These need to advertise as localhost. --- .../pulsar/broker/auth/MockedPulsarServiceBaseTest.java | 1 + .../java/com/yahoo/pulsar/broker/web/WebServiceTest.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index ad636268d4be6..af7bdc01a6a40 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -77,6 +77,7 @@ public MockedPulsarServiceBaseTest() { this.conf.setWebServicePort(BROKER_WEBSERVICE_PORT); this.conf.setWebServicePortTls(BROKER_WEBSERVICE_PORT_TLS); this.conf.setClusterName("test"); + this.conf.setAdvertisedAddress("localhost"); // there are TLS tests in here, they need to use localhost because of the certificate } protected final void internalSetup() throws Exception { diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java index 2963f09b5c8ea..dfc78be7713be 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/web/WebServiceTest.java @@ -70,8 +70,8 @@ public class WebServiceTest { private PulsarService pulsar; private final static int BROKER_WEBSERVICE_PORT = PortManager.nextFreePort(); private final static int BROKER_WEBSERVICE_PORT_TLS = PortManager.nextFreePort(); - private static final String BROKER_URL_BASE = "http://" + AddressUtils.unsafeLocalHostName() + ":" + BROKER_WEBSERVICE_PORT; - private static final String BROKER_URL_BASE_TLS = "https://" + AddressUtils.unsafeLocalHostName() + ":" + BROKER_WEBSERVICE_PORT_TLS; + private static final String BROKER_URL_BASE = "http://localhost:" + BROKER_WEBSERVICE_PORT; + private static final String BROKER_URL_BASE_TLS = "https://localhost:" + BROKER_WEBSERVICE_PORT_TLS; private static final String BROKER_LOOKUP_URL = BROKER_URL_BASE + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic"; private static final String BROKER_LOOKUP_URL_TLS = BROKER_URL_BASE_TLS @@ -290,6 +290,7 @@ private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowU config.setTlsAllowInsecureConnection(allowInsecure); config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH); config.setClusterName("local"); + config.setAdvertisedAddress("localhost"); // TLS certificate expects localhost pulsar = spy(new PulsarService(config)); doReturn(new MockedZooKeeperClientFactoryImpl()).when(pulsar).getZooKeeperClientFactory(); pulsar.start(); From 18d3c912aa5c9da40476dfb5ce2e0a8a78686922 Mon Sep 17 00:00:00 2001 From: radekg Date: Sat, 24 Sep 2016 02:42:42 +0200 Subject: [PATCH 13/18] Missing license header. --- .../java/com/yahoo/pulsar/utils/AddressUtils.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java index 5147743f9d25d..c8171a854faf9 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java @@ -1,3 +1,18 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.yahoo.pulsar.utils; import java.net.Inet4Address; From 2683759db68f29d39bcf1ad241eb3a913c8688b0 Mon Sep 17 00:00:00 2001 From: radekg Date: Sat, 24 Sep 2016 03:36:28 +0200 Subject: [PATCH 14/18] Fixes BrokerServiceTest.testBrokerStatsMetrics test. --- .../com/yahoo/pulsar/broker/service/BrokerServiceTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java index 3260239485879..bf973ef63d0c0 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java @@ -213,11 +213,8 @@ public void testBrokerStatsMetrics() throws Exception { JSONArray metrics = brokerStatsClient.getMetrics(); assertEquals(metrics.length(), 4, metrics.toString()); - JSONObject obj = metrics.getJSONObject(2); - assertTrue(obj.getString("dimensions").contains("prop/use/ns-abc")); - - obj = metrics.getJSONObject(1); - assertTrue(obj.getString("dimensions").contains("topic_load_times")); + assertTrue(metrics.getJSONObject(1).getString("dimensions").contains("prop/use/ns-abc")); + assertTrue(metrics.getJSONObject(2).getString("dimensions").contains("topic_load_times")); Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT); } From 1db4d342b0e590c68804b5e3a3895226996d9036 Mon Sep 17 00:00:00 2001 From: radekg Date: Sat, 24 Sep 2016 03:48:57 +0200 Subject: [PATCH 15/18] Updates comments for advertisedAddress in the configuration files. --- conf/broker.conf | 6 +++--- conf/standalone.conf | 6 +++--- conf/websocket.conf | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/conf/broker.conf b/conf/broker.conf index 151995dda89c3..d028b4cc452dc 100644 --- a/conf/broker.conf +++ b/conf/broker.conf @@ -37,10 +37,10 @@ webServicePortTls=8443 # Enable the WebSocket API service in broker webSocketServiceEnabled=false -# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). -bindAddress= +# Hostname or IP address the service binds on, default is 0.0.0.0. +bindAddress=0.0.0.0 -# Hostname or IP address the service advertises to the outside world. If not set, the value of bindAddress is used. +# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. advertisedAddress= # Name of the cluster to which this broker belongs to diff --git a/conf/standalone.conf b/conf/standalone.conf index 6f68da6087f7a..379cb0ac1d33b 100644 --- a/conf/standalone.conf +++ b/conf/standalone.conf @@ -27,10 +27,10 @@ brokerServicePort=6650 # Port to use to server HTTP request webServicePort=8080 -# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). -bindAddress= +# Hostname or IP address the service binds on, default is 0.0.0.0. +bindAddress=0.0.0.0 -# Hostname or IP address the service advertises to the outside world, default is InetAddress.getLocalHost().getHostName(). +# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used. advertisedAddress= # Name of the cluster to which this broker belongs to diff --git a/conf/websocket.conf b/conf/websocket.conf index c30d461f05869..1dc70cd719fe5 100644 --- a/conf/websocket.conf +++ b/conf/websocket.conf @@ -22,8 +22,8 @@ globalZookeeperServers= # Port to use to server HTTP request webServicePort=8080 -# Hostname or IP address the service binds on, default is InetAddress.getLocalHost().getHostName(). -bindAddress= +# Hostname or IP address the service binds on, default is 0.0.0.0. +bindAddress=0.0.0.0 # Name of the pulsar cluster to connect to clusterName= From e51b961d543f1c929f4f0d3513aa5e527cdc902c Mon Sep 17 00:00:00 2001 From: radekg Date: Mon, 26 Sep 2016 16:39:15 +0200 Subject: [PATCH 16/18] Unit tests passing. --- .../src/main/java/com/yahoo/pulsar/broker/PulsarService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java index 90fd9e857f619..006ee1a8aae97 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java @@ -322,7 +322,7 @@ private void acquireSLANamespace() { try { // Namespace not created hence no need to unload it if (!this.globalZkCache.exists( - AdminResource.path("policies") + "/" + NamespaceService.getSLAMonitorNamespace(host, config))) { + AdminResource.path("policies") + "/" + NamespaceService.getSLAMonitorNamespace(getAdvertisedAddress(), config))) { return; } if (!this.nsservice.registerSLANamespace()) { From 1ac44bf262d17097f660e88a72667cdbca773bef Mon Sep 17 00:00:00 2001 From: radekg Date: Mon, 26 Sep 2016 16:53:04 +0200 Subject: [PATCH 17/18] Last place where the getHost() needs to be replaced with getAdvertisedAddress() --- .../java/com/yahoo/pulsar/broker/admin/PersistentTopics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java index 0e8deb30e3b31..bb9e04c5ab7dc 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/admin/PersistentTopics.java @@ -903,7 +903,7 @@ public PersistentOfflineTopicStats getBacklog(@PathParam("property") String prop } final ManagedLedgerConfig config = pulsar().getBrokerService().getManagedLedgerConfig(dn).get(); ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(config.getDigestType(), - config.getPassword(), pulsar().getHost(), false); + config.getPassword(), pulsar().getAdvertisedAddress(), false); offlineTopicStats = offlineTopicBacklog .estimateUnloadedTopicBacklog((ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), dn); pulsar().getBrokerService().cacheOfflineTopicStats(dn, offlineTopicStats); From d4c76f5eb8abb653ae0032c180027a1714dd95fb Mon Sep 17 00:00:00 2001 From: radekg Date: Mon, 26 Sep 2016 23:35:38 +0200 Subject: [PATCH 18/18] Address the final review point, adjust the metrics test, rename PulsarService.getHost() to PulsarService.getBindAddress(). --- .../broker/ServiceConfigurationUtils.java | 44 +++++++++++++++++ .../yahoo/pulsar/broker/PulsarService.java | 49 +++++-------------- .../pulsar/broker/service/BrokerService.java | 8 +-- .../pulsar/broker/service/PulsarStats.java | 2 +- .../yahoo/pulsar/broker/web/WebService.java | 4 +- .../broker/namespace/OwnershipCacheTest.java | 2 - .../broker/service/BrokerServiceTest.java | 20 +++++++- .../pulsar/broker/web/WebServiceTest.java | 1 - .../com/yahoo/pulsar/utils/AddressUtils.java | 31 ------------ 9 files changed, 79 insertions(+), 82 deletions(-) create mode 100644 pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfigurationUtils.java delete mode 100644 pulsar-broker/src/test/java/com/yahoo/pulsar/utils/AddressUtils.java diff --git a/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfigurationUtils.java b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfigurationUtils.java new file mode 100644 index 0000000000000..a7c0141eafe51 --- /dev/null +++ b/pulsar-broker-common/src/main/java/com/yahoo/pulsar/broker/ServiceConfigurationUtils.java @@ -0,0 +1,44 @@ +/** + * Copyright 2016 Yahoo Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.yahoo.pulsar.broker; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +public class ServiceConfigurationUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ServiceConfigurationUtils.class); + + public static String getDefaultOrConfiguredAddress(String configuredAddress) { + if ( configuredAddress == null ) { + return unsafeLocalhostResolve(); + } + return configuredAddress; + } + + public static String unsafeLocalhostResolve() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException ex) { + LOG.error(ex.getMessage(), ex); + throw new IllegalStateException("Failed to resolve localhost name.", ex); + } + } + +} diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java index 006ee1a8aae97..045fdc82e2cfd 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/PulsarService.java @@ -98,7 +98,7 @@ public class PulsarService implements AutoCloseable { private LoadManager loadManager = null; private PulsarAdmin adminClient = null; private ZooKeeperClientFactory zkClientFactory = null; - private final String host; + private final String bindAddress; private final String advertisedAddress; private final String webServiceAddress; private final String webServiceAddressTls; @@ -120,8 +120,8 @@ public enum State { public PulsarService(ServiceConfiguration config) { state = State.Init; - this.host = host(config); - this.advertisedAddress = advertisedHost(config); + this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getBindAddress()); + this.advertisedAddress = advertisedAddress(config); this.webServiceAddress = webAddress(config); this.webServiceAddressTls = webAddressTls(config); this.brokerServiceUrl = brokerUrl(config); @@ -565,65 +565,40 @@ public MessagingServiceShutdownHook getShutdownService() { } /** - * Derive the host - * - * @return Hostname or IP address the service binds on. - */ - public static String host(ServiceConfiguration config) { - if (config.getBindAddress() == null) { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException ex) { - LOG.error(ex.getMessage(), ex); - throw new IllegalStateException("Failed to resolve localhost name.", ex); - } - } - return config.getBindAddress(); - } - - /** - * Advertised service host. + * Advertised service address. * * @return Hostname or IP address the service advertises to the outside world. */ - public static String advertisedHost(ServiceConfiguration config) { - if (config.getAdvertisedAddress() == null) { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException ex) { - LOG.error(ex.getMessage(), ex); - throw new IllegalStateException("Failed to resolve localhost name.", ex); - } - } - return config.getAdvertisedAddress(); + public static String advertisedAddress(ServiceConfiguration config) { + return ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress()); } public static String brokerUrl(ServiceConfiguration config) { - return "pulsar://" + advertisedHost(config) + ":" + config.getBrokerServicePort(); + return "pulsar://" + advertisedAddress(config) + ":" + config.getBrokerServicePort(); } public static String brokerUrlTls(ServiceConfiguration config) { if (config.isTlsEnabled()) { - return "pulsar://" + advertisedHost(config) + ":" + config.getBrokerServicePortTls(); + return "pulsar://" + advertisedAddress(config) + ":" + config.getBrokerServicePortTls(); } else { return ""; } } public static String webAddress(ServiceConfiguration config) { - return String.format("http://%s:%d", advertisedHost(config), config.getWebServicePort()); + return String.format("http://%s:%d", advertisedAddress(config), config.getWebServicePort()); } public static String webAddressTls(ServiceConfiguration config) { if (config.isTlsEnabled()) { - return String.format("https://%s:%d", advertisedHost(config), config.getWebServicePortTls()); + return String.format("https://%s:%d", advertisedAddress(config), config.getWebServicePortTls()); } else { return ""; } } - public String getHost() { - return host; + public String getBindAddress() { + return bindAddress; } public String getAdvertisedAddress() { diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java index 4ed0a0654f100..ecc460ded45dd 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/BrokerService.java @@ -29,15 +29,12 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import org.apache.bookkeeper.client.BookKeeper.DigestType; @@ -73,7 +70,6 @@ import com.yahoo.pulsar.common.naming.DestinationName; import com.yahoo.pulsar.common.naming.NamespaceBundle; import com.yahoo.pulsar.common.naming.NamespaceBundleFactory; -import com.yahoo.pulsar.common.naming.NamespaceBundles; import com.yahoo.pulsar.common.naming.NamespaceName; import com.yahoo.pulsar.common.naming.ServiceUnitId; import com.yahoo.pulsar.common.policies.data.ClusterData; @@ -221,13 +217,13 @@ public void start() throws Exception { bootstrap.childHandler(new PulsarChannelInitializer(this, serviceConfig, false)); // Bind and start to accept incoming connections. - bootstrap.bind(new InetSocketAddress(pulsar.getHost(), port)).sync(); + bootstrap.bind(new InetSocketAddress(pulsar.getBindAddress(), port)).sync(); log.info("Started Pulsar Broker service on port {}", port); if (serviceConfig.isTlsEnabled()) { ServerBootstrap tlsBootstrap = bootstrap.clone(); tlsBootstrap.childHandler(new PulsarChannelInitializer(this, serviceConfig, true)); - tlsBootstrap.bind(new InetSocketAddress(pulsar.getHost(), tlsPort)).sync(); + tlsBootstrap.bind(new InetSocketAddress(pulsar.getBindAddress(), tlsPort)).sync(); log.info("Started Pulsar Broker TLS service on port {}", tlsPort); } diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java index fdaad39c55be7..4caa13b2c23ea 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/service/PulsarStats.java @@ -66,7 +66,7 @@ public PulsarStats(PulsarService pulsar) { this.tempMetricsCollection = Lists.newArrayList(); this.metricsCollection = Lists.newArrayList(); this.brokerOperabilityMetrics = new BrokerOperabilityMetrics(pulsar.getConfiguration().getClusterName(), - pulsar.getHost()); + pulsar.getAdvertisedAddress()); } @Override diff --git a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java index f86716dccd5c9..93a9e5d558db0 100644 --- a/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java +++ b/pulsar-broker/src/main/java/com/yahoo/pulsar/broker/web/WebService.java @@ -85,7 +85,7 @@ public WebService(ServiceConfiguration config, PulsarService pulsar) throws Puls ServerConnector connector = new PulsarServerConnector(server, 1, 1); connector.setPort(config.getWebServicePort()); - connector.setHost(pulsar.getHost()); + connector.setHost(pulsar.getBindAddress()); connectors.add(connector); if (config.isTlsEnabled()) { @@ -103,7 +103,7 @@ public WebService(ServiceConfiguration config, PulsarService pulsar) throws Puls sslCtxFactory.setWantClientAuth(true); ServerConnector tlsConnector = new PulsarServerConnector(server, 1, 1, sslCtxFactory); tlsConnector.setPort(config.getWebServicePortTls()); - tlsConnector.setHost(pulsar.getHost()); + tlsConnector.setHost(pulsar.getBindAddress()); connectors.add(tlsConnector); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java index 8afb27c6f0616..f1c8cdc70eafa 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/namespace/OwnershipCacheTest.java @@ -16,7 +16,6 @@ package com.yahoo.pulsar.broker.namespace; import static com.google.common.base.Preconditions.checkNotNull; -import static com.yahoo.pulsar.broker.PulsarService.host; import static com.yahoo.pulsar.broker.PulsarService.webAddress; import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.doReturn; @@ -80,7 +79,6 @@ public void setup() throws Exception { doReturn(nsService).when(pulsar).getNamespaceService(); doReturn(port).when(config).getBrokerServicePort(); doReturn(brokerService).when(pulsar).getBrokerService(); - doReturn(host(config)).when(pulsar).getAdvertisedAddress(); doReturn(webAddress(config)).when(pulsar).getWebServiceAddress(); doReturn(selfBrokerUrl).when(pulsar).getBrokerServiceUrl(); } diff --git a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java index bf973ef63d0c0..4288b7e2a400d 100644 --- a/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java +++ b/pulsar-broker/src/test/java/com/yahoo/pulsar/broker/service/BrokerServiceTest.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -213,8 +214,23 @@ public void testBrokerStatsMetrics() throws Exception { JSONArray metrics = brokerStatsClient.getMetrics(); assertEquals(metrics.length(), 4, metrics.toString()); - assertTrue(metrics.getJSONObject(1).getString("dimensions").contains("prop/use/ns-abc")); - assertTrue(metrics.getJSONObject(2).getString("dimensions").contains("topic_load_times")); + // these metrics seem to be arriving in different order at different times... + // is the order really relevant here? + boolean namespaceDimensionFound = false; + boolean topicLoadTimesDimensionFound = false; + for ( int i=0; i