Merge pull request #1841 from ClickHouse/clientv2_custom_headers

[client-v2] Added options for additional headers and server settings

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNodes.java

@@ -68,6 +68,7 @@ static ClickHouseNodes create(String endpoints, Map<?, ?> defaultOptions) {
         } else {
             index = 0;
         }
+
 
         String defaultParams = "";
         Set<String> list = new LinkedHashSet<>();
@@ -95,14 +96,19 @@ static ClickHouseNodes create(String endpoints, Map<?, ?> defaultOptions) {
             }
 
             int endIndex = i;
+            // parsing host name
             for (int j = i + 1; j < len; j++) {
                 ch = endpoints.charAt(j);
                 if (ch == stopChar || Character.isWhitespace(ch)) {
                     endIndex = j;
                     break;
+                } else if ( stopChar == ',' && ( ch == '/' || ch == '?' || ch == '#') ) {
+                    break;
                 }
             }
+
             if (endIndex > i) {
+                // add host name to list
                 list.add(endpoints.substring(index, endIndex).trim());
                 i = endIndex;
                 index = endIndex + 1;

clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java

@@ -1,6 +1,8 @@
 package com.clickhouse.client;
 
+import com.clickhouse.config.ClickHouseOption;
 import org.testng.Assert;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.net.URI;
@@ -438,4 +440,34 @@ public void testToUri() {
                         .toUri().toString(),
                 "http://server1.dc1:8123/db1?async=false&auto_discovery=true#apj,r1s1");
     }
+
+    @Test(groups = { "unit" }, dataProvider = "testPropertyWithValueList_endpoints")
+    public void testPropertyWithValueList(String endpoints, int numOfNodes, String[] expectedBaseUris) {
+        ClickHouseNodes node = ClickHouseNodes.of(endpoints);
+        Assert.assertEquals(node.nodes.size(), numOfNodes, "Number of nodes does not match");
+
+        int i = 0;
+        for (ClickHouseNode n : node.nodes) {
+            Assert.assertEquals(n.config.getDatabase(), "my_db");
+            Assert.assertEquals(expectedBaseUris[i++], n.getBaseUri());
+            String customSettings = (String)n.config.getOption(ClickHouseClientOption.CUSTOM_SETTINGS);
+            String configSettings = (String) n.config.getOption(ClickHouseClientOption.CUSTOM_SETTINGS);
+
+            Arrays.asList(customSettings, configSettings).forEach((settings) -> {
+                Map<String, String> settingsMap = ClickHouseOption.toKeyValuePairs(settings);
+                Assert.assertEquals(settingsMap.get("param1"), "value1");
+                Assert.assertEquals(settingsMap.get("param2"), "value2");
+            });
+        }
+    }
+
+    @DataProvider(name = "testPropertyWithValueList_endpoints")
+    public static Object[][] endpoints() {
+        return new Object[][] {
+            { "http://server1:9090/my_db?custom_settings=param1=value1,param2=value2", 1, new String[]{"http://server1:9090/"} },
+            { "http://server1/my_db?custom_settings=param1=value1,param2=value2", 1, new String[]{"http://server1:8123/"} },
+            { "http://server1:9090,server2/my_db?custom_settings=param1=value1,param2=value2", 2, new String[]{"http://server1:9090/", "http://server2:8123/"} },
+            { "http://server1,server2:9090/my_db?custom_settings=param1=value1,param2=value2", 2, new String[]{"http://server1:8123/", "http://server2:9090/"} }
+        };
+    }
 }

clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java

@@ -214,8 +214,9 @@ public void testNodeGrouping() throws ExecutionException, InterruptedException,
         Assert.assertEquals(nodes.get(), ClickHouseNode.of("tcp://b:9000/test?x=1"));
     }
 
-    @Test(groups = { "unit" })
+    @Test(groups = { "unit" }, enabled = false)
     public void testQueryWithSlash() {
+        // test is disabled because this format of urls is not supported
         ClickHouseNodes servers = ClickHouseNodes
                 .of("https://node1?a=/b/c/d,node2/db2?/a/b/c=d,node3/db1?a=/d/c.b");
         Assert.assertEquals(servers.nodes.get(0).getDatabase().orElse(null), "db1");
@@ -268,30 +269,33 @@ public void testMultiNodeList() {
         Assert.assertEquals(ClickHouseNodes.of("http://(a) , {b}, [::1]"), new ClickHouseNodes(
                 Arrays.asList(ClickHouseNode.of("http://a"), ClickHouseNode.of("http://b"),
                         ClickHouseNode.of("http://[::1]"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c"), new ClickHouseNodes(
-                Arrays.asList(ClickHouseNode.of("http://a"), ClickHouseNode.of("tcp://b"),
-                        ClickHouseNode.of("grpc://c"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c/"), new ClickHouseNodes(
-                Arrays.asList(ClickHouseNode.of("http://a"), ClickHouseNode.of("tcp://b"),
-                        ClickHouseNode.of("grpc://c"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c/db1"), new ClickHouseNodes(
-                Arrays.asList(ClickHouseNode.of("http://a/db1"), ClickHouseNode.of("tcp://b/db1"),
-                        ClickHouseNode.of("grpc://c/db1"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c?a=1"), new ClickHouseNodes(
-                Arrays.asList(ClickHouseNode.of("http://a?a=1"), ClickHouseNode.of("tcp://b?a=1"),
-                        ClickHouseNode.of("grpc://c?a=1"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c#dc1"), new ClickHouseNodes(
-                Arrays.asList(ClickHouseNode.of("http://a#dc1"), ClickHouseNode.of("tcp://b#dc1"),
-                        ClickHouseNode.of("grpc://c#dc1"))));
-        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c:1234/some/db?a=1#dc1"),
-                new ClickHouseNodes(
-                        Arrays.asList(ClickHouseNode.of("http://a/some/db?a=1#dc1"),
-                                ClickHouseNode.of("tcp://b/some/db?a=1#dc1"),
-                                ClickHouseNode.of("grpc://c:1234/some/db?a=1#dc1"))));
+
+        // THIS IS SHOULD NOT BE SUPPORTED
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c"), new ClickHouseNodes(
+//                Arrays.asList(ClickHouseNode.of("http://a"), ClickHouseNode.of("tcp://b"),
+//                        ClickHouseNode.of("grpc://c"))));
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c/"), new ClickHouseNodes(
+//                Arrays.asList(ClickHouseNode.of("http://a"), ClickHouseNode.of("tcp://b"),
+//                        ClickHouseNode.of("grpc://c"))));
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c/db1"), new ClickHouseNodes(
+//                Arrays.asList(ClickHouseNode.of("http://a/db1"), ClickHouseNode.of("tcp://b/db1"),
+//                        ClickHouseNode.of("grpc://c/db1"))));
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c?a=1"), new ClickHouseNodes(
+//                Arrays.asList(ClickHouseNode.of("http://a?a=1"), ClickHouseNode.of("tcp://b?a=1"),
+//                        ClickHouseNode.of("grpc://c?a=1"))));
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c#dc1"), new ClickHouseNodes(
+//                Arrays.asList(ClickHouseNode.of("http://a#dc1"), ClickHouseNode.of("tcp://b#dc1"),
+//                        ClickHouseNode.of("grpc://c#dc1"))));
+//        Assert.assertEquals(ClickHouseNodes.of("http://a,tcp://b,grpc://c:1234/some/db?a=1#dc1"),
+//                new ClickHouseNodes(
+//                        Arrays.asList(ClickHouseNode.of("http://a/some/db?a=1#dc1"),
+//                                ClickHouseNode.of("tcp://b/some/db?a=1#dc1"),
+//                                ClickHouseNode.of("grpc://c:1234/some/db?a=1#dc1"))));
     }
 
-    @Test(groups = { "unit" })
+    @Test(groups = { "unit" }, enabled = false)
     public void testManageAndUnmanageNewNode() {
+        // test is disabled because this format of urls is not supported
         ClickHouseNodes nodes = ClickHouseNodes.create("https://a,grpcs://b,mysql://c", null);
         Assert.assertEquals(nodes.getPolicy(), ClickHouseLoadBalancingPolicy.DEFAULT);
         Assert.assertEquals(nodes.nodes.size(), 3);
@@ -322,7 +326,7 @@ public void testManageAndUnmanageNewNode() {
 
     @Test(groups = { "unit" })
     public void testManageAndUnmanageSameNode() {
-        ClickHouseNodes nodes = ClickHouseNodes.create("grpc://(http://a), tcp://b, c", null);
+        ClickHouseNodes nodes = ClickHouseNodes.create("http://a,b,c", null);
         Assert.assertEquals(nodes.nodes.size(), 3);
         Assert.assertEquals(nodes.faultyNodes.size(), 0);
         ClickHouseNode node = ClickHouseNode.of("http://a");
@@ -337,7 +341,7 @@ public void testManageAndUnmanageSameNode() {
         Assert.assertEquals(nodes.faultyNodes.size(), 0);
 
         // now repeat same scenario but using different method
-        node = ClickHouseNode.of("tcp://b");
+        node = ClickHouseNode.of("http://b");
         Assert.assertTrue(node.isStandalone(), "Newly created node is always standalone");
         node.setManager(nodes);
         Assert.assertTrue(node.isManaged(), "Node should be managed");

clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java

@@ -20,9 +20,7 @@ public enum ClickHouseHttpOption implements ClickHouseOption {
      */
     CUSTOM_HEADERS("custom_http_headers", "", "Custom HTTP headers."),
     /**
-     * Custom HTTP query parameters. Consider
-     * {@link com.clickhouse.client.config.ClickHouseClientOption#CUSTOM_SETTINGS}
-     * if you don't want your implementation ties to http protocol.
+     * @deprecated use {@link com.clickhouse.client.config.ClickHouseClientOption#CUSTOM_SETTINGS}
      */
     CUSTOM_PARAMS("custom_http_params", "", "Custom HTTP query parameters."),
     /**

clickhouse-jdbc/pom.xml

@@ -142,6 +142,12 @@
             <artifactId>mysql-connector-j</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-http-client</artifactId>
+            <version>0.6.5-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <profiles>

clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseDriverTest.java

@@ -4,6 +4,7 @@
 
 import com.clickhouse.client.ClickHouseProtocol;
 
+import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 

clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java

@@ -6,12 +6,14 @@
 import java.util.Properties;
 
 import com.clickhouse.client.ClickHouseCredentials;
+import com.clickhouse.client.ClickHouseLoadBalancingPolicy;
 import com.clickhouse.client.ClickHouseNode;
 import com.clickhouse.client.ClickHouseProtocol;
 import com.clickhouse.client.config.ClickHouseDefaults;
 import com.clickhouse.jdbc.internal.ClickHouseJdbcUrlParser.ConnectionInfo;
 
 import org.testng.Assert;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class ClickHouseJdbcUrlParserTest {
@@ -154,4 +156,24 @@ public void testParseCredentials() throws SQLException {
         Assert.assertEquals(server.getCredentials().get().getUserName(), "let@me:in");
         Assert.assertEquals(server.getCredentials().get().getPassword(), "let@me:in");
     }
+
+    @Test(groups = "unit", dataProvider = "testParseUrlPropertiesProvider")
+    public void testParseUrlProperties(String url, int numOfNodes) throws SQLException {
+
+        ConnectionInfo info = ClickHouseJdbcUrlParser.parse(url, null);
+        Assert.assertEquals(info.getNodes().getNodes().size(), numOfNodes);
+        Assert.assertEquals(info.getNodes().getPolicy().getClass().getSimpleName(), "FirstAlivePolicy");
+        for (ClickHouseNode n : info.getNodes().getNodes()) {
+            Assert.assertEquals(n.getOptions().get("connect_timeout"), "10000");
+            Assert.assertEquals(n.getOptions().get("http_connection_provider"), "HTTP_CLIENT");
+        }
+    }
+
+    @DataProvider(name = "testParseUrlPropertiesProvider")
+    public static Object[][] testParseUrlPropertiesProvider() {
+        return new Object[][] {
+                { "jdbc:clickhouse://host1:8123,host2:8123,host3:8123/db1?http_connection_provider=HTTP_CLIENT&load_balancing_policy=firstAlive&connect_timeout=10000", 3 },
+                { "jdbc:clickhouse:http://host1:8123,host2:8123,host3:8123/db1?http_connection_provider=HTTP_CLIENT&load_balancing_policy=firstAlive&connect_timeout=10000", 3 }
+        };
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -66,6 +66,7 @@
 import java.time.ZoneId;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -120,6 +121,7 @@
  *
  */
 public class Client implements AutoCloseable {
+
     private HttpAPIClientHelper httpClientHelper = null;
 
     private final Set<String> endpoints;
@@ -778,6 +780,72 @@ public Builder allowBinaryReaderToReuseBuffers(boolean reuse) {
             return this;
         }
 
+        /**
+         * Defines list of headers that should be sent with each request. The Client will use a header value
+         * defined in {@code headers} instead of any other.
+         * Operation settings may override these headers.
+         *
+         * @see InsertSettings#httpHeaders(Map)
+         * @see QuerySettings#httpHeaders(Map)
+         * @see CommandSettings#httpHeaders(Map)
+         * @param key - a name of the header.
+         * @param value - a value of the header.
+         * @return same instance of the builder
+         */
+        public Builder httpHeader(String key, String value) {
+            this.configuration.put(ClientSettings.HTTP_HEADER_PREFIX + key, value);
+            return this;
+        }
+
+        /**
+         * {@see #httpHeader(String, String)} but for multiple values.
+         * @param key - name of the header
+         * @param values - collection of values
+         * @return same instance of the builder
+         */
+        public Builder httpHeader(String key, Collection<String> values) {
+            this.configuration.put(ClientSettings.HTTP_HEADER_PREFIX + key, ClientSettings.commaSeparated(values));
+            return this;
+        }
+
+        /**
+         * {@see #httpHeader(String, String)} but for multiple headers.
+         * @param headers - map of headers
+         * @return same instance of the builder
+         */
+        public Builder httpHeaders(Map<String, String> headers) {
+            headers.forEach(this::httpHeader);
+            return this;
+        }
+
+        /**
+         * Defines list of server settings that should be sent with each request. The Client will use a setting value
+         * defined in {@code settings} instead of any other.
+         * Operation settings may override these values.
+         *
+         * @see InsertSettings#serverSetting(String, String) (Map)
+         * @see QuerySettings#serverSetting(String, String) (Map)
+         * @see CommandSettings#serverSetting(String, String) (Map)
+         * @param name - name of the setting without special prefix
+         * @param value - value of the setting
+         * @return same instance of the builder
+         */
+        public Builder serverSetting(String name, String value) {
+            this.configuration.put(ClientSettings.SERVER_SETTING_PREFIX + name, value);
+            return this;
+        }
+
+        /**
+         * {@see #serverSetting(String, String)} but for multiple values.
+         * @param name - name of the setting without special prefix
+         * @param values - collection of values
+         * @return same instance of the builder
+         */
+        public Builder serverSetting(String name,  Collection<String> values) {
+            this.configuration.put(ClientSettings.SERVER_SETTING_PREFIX + name, ClientSettings.commaSeparated(values));
+            return this;
+        }
+
         public Client build() {
             setDefaults();
 

client-v2/src/main/java/com/clickhouse/client/api/ClientSettings.java

@@ -0,0 +1,31 @@
+package com.clickhouse.client.api;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * All known client settings at current version.
+ *
+ */
+public class ClientSettings {
+
+    public static final String HTTP_HEADER_PREFIX = "http_header_";
+
+    public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_";
+
+    public static String commaSeparated(Collection<?> values) {
+        StringBuilder sb = new StringBuilder();
+        for (Object value : values) {
+            sb.append(value.toString().replaceAll(",", "\\,")).append(",");
+        }
+        sb.setLength(sb.length() - 1);
+        return sb.toString();
+    }
+
+    public static List<String> valuesFromCommaSeparated(String value) {
+        return Arrays.stream(value.split(",")).map(s -> s.replaceAll("\\\\,", ","))
+                .collect(Collectors.toList());
+    }
+}