Merge pull request #142 from datafuselabs/fix/str-null2-null

fix: get null value

databend-client/src/main/java/com/databend/client/DatabendClientV1.java

@@ -41,13 +41,13 @@
 @ThreadSafe
 public class DatabendClientV1
         implements DatabendClient {
-    private static final String USER_AGENT_VALUE = DatabendClientV1.class.getSimpleName() +
+    public static final String USER_AGENT_VALUE = DatabendClientV1.class.getSimpleName() +
             "/" +
             firstNonNull(DatabendClientV1.class.getPackage().getImplementationVersion(), "jvm-unknown");
-    private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
-    private static final JsonCodec<QueryResults> QUERY_RESULTS_CODEC = jsonCodec(QueryResults.class);
+    public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8");
+    public static final JsonCodec<QueryResults> QUERY_RESULTS_CODEC = jsonCodec(QueryResults.class);
 
-    private static final String QUERY_PATH = "/v1/query";
+    public static final String QUERY_PATH = "/v1/query";
     private static final long MAX_MATERIALIZED_JSON_RESPONSE_SIZE = 128 * 1024;
     private final OkHttpClient httpClient;
     private final String query;
@@ -222,7 +222,7 @@ public boolean isRunning() {
     }
 
     @Override
-    public Map<String, String> getAdditionalHeaders() {
+    public  Map<String, String> getAdditionalHeaders() {
         return additonalHeaders;
     }
 

databend-jdbc/src/main/java/com/databend/jdbc/AbstractDatabendResultSet.java

@@ -356,8 +356,13 @@ private Object column(int index)
         if ((index <= 0) || (index > resultSetMetaData.getColumnCount())) {
             throw new SQLException("Invalid column index: " + index);
         }
-        Object value = row.get().get(index - 1);
-        wasNull.set(value == null);
+        Object value = null;
+        value = row.get().get(index - 1);
+        if (value == null || value.toString().equals("NULL")) {
+            wasNull.set(value == null);
+            return null;
+        }
+
         return value;
     }
 

databend-jdbc/src/main/java/com/databend/jdbc/ConnectionProperties.java

@@ -15,6 +15,7 @@ public final class ConnectionProperties {
     public static final ConnectionProperty<String> USER = new User();
     public static final ConnectionProperty<String> PASSWORD = new Password();
     public static final ConnectionProperty<Boolean> SSL = new Ssl();
+    public static final ConnectionProperty<Boolean> STRNULL_AS_NULL = new StrNullAsNull();
     public static final ConnectionProperty<String> WAREHOUSE = new Warehouse();
     public static final ConnectionProperty<String> SSL_MODE = new SSLMode();
     static final ConnectionProperty<String> TENANT = new Tenant();
@@ -39,6 +40,7 @@ public final class ConnectionProperties {
             .add(USER)
             .add(PASSWORD)
             .add(SSL)
+            .add(STRNULL_AS_NULL)
             .add(WAREHOUSE)
             .add(SSL_MODE)
             .add(TENANT)
@@ -84,6 +86,13 @@ public Ssl() {
         }
     }
 
+    public static class StrNullAsNull
+            extends AbstractConnectionProperty<Boolean> {
+        public StrNullAsNull() {
+            super("strnullasnull", Optional.of("true"), NOT_REQUIRED, ALLOWED, BOOLEAN_CONVERTER);
+        }
+    }
+
     private static class Database
             extends AbstractConnectionProperty<String> {
         public Database() {

databend-jdbc/src/main/java/com/databend/jdbc/DatabendConnection.java

@@ -1,18 +1,15 @@
 package com.databend.jdbc;
 
-import com.databend.client.ClientSettings;
-import com.databend.client.DatabendClient;
-import com.databend.client.DatabendClientV1;
-import com.databend.client.DatabendSession;
-import com.databend.client.PaginationOptions;
-import com.databend.client.StageAttachment;
+import com.databend.client.*;
 import com.databend.jdbc.annotation.NotImplemented;
 import com.databend.jdbc.cloud.DatabendCopyParams;
 import com.databend.jdbc.cloud.DatabendPresignClient;
 import com.databend.jdbc.cloud.DatabendPresignClientV1;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import okhttp3.Headers;
+import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
+import okhttp3.Request;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -43,7 +40,9 @@
 import java.util.logging.Logger;
 
 import static com.databend.client.ClientSettings.*;
+import static com.databend.client.DatabendClientV1.*;
 import static com.google.common.base.Preconditions.checkState;
+import static java.net.HttpURLConnection.HTTP_OK;
 import static java.util.Collections.newSetFromMap;
 import static java.util.Objects.requireNonNull;
 
@@ -492,6 +491,10 @@ public String warehouse() {
         return this.driverUri.getWarehouse();
     }
 
+    public Boolean strNullAsNull() {
+        return this.driverUri.getStrNullAsNull();
+    }
+
     public String tenant() {
         return this.driverUri.getTenant();
     }
@@ -516,23 +519,65 @@ public URI getURI() {
         return this.httpUri;
     }
 
+    public void PingDatabendClientV1() throws IOException {
+        ClientSettings settings = makeClientSettings();
+        String query = "select 1";
+        HttpUrl url = HttpUrl.get(settings.getHost());
+        QueryRequest req = QueryRequest.builder().setSession(settings.getSession()).setStageAttachment(settings.getStageAttachment()).setPaginationOptions(settings.getPaginationOptions()).setSql(query).build();
+        String reqString = req.toString();
+        if (reqString == null || reqString.isEmpty()) {
+            throw new IllegalArgumentException("Invalid request: " + req);
+        }
+        url = url.newBuilder().encodedPath(QUERY_PATH).build();
+        Request.Builder builder = new Request.Builder()
+                .url(url)
+                .header("User-Agent", USER_AGENT_VALUE)
+                .header("Accept", "application/json")
+                .header("Content-Type", "application/json");
+        if (settings.getAdditionalHeaders() != null) {
+            settings.getAdditionalHeaders().forEach(builder::addHeader);
+        }
+        Request request = builder.post(okhttp3.RequestBody.create(MEDIA_TYPE_JSON, reqString)).build();
+        executePing(request);
+    }
+
+
+    public void executePing(Request request) throws IOException {
+        requireNonNull(request, "request is null");
+        try {
+            JsonResponse<QueryResults> response = JsonResponse.execute(QUERY_RESULTS_CODEC, httpClient, request, OptionalLong.empty());
+            if ((response.getStatusCode() < 400)) {
+                return;
+            } else {
+                throw new IOException("failed to ping databend server");
+            }
+        } catch (RuntimeException e) {
+            throw new IOException(e);
+        }
+    }
+
     // TODO(zhihanz): session property push down
     DatabendClient startQuery(String sql) throws SQLException {
+        return new DatabendClientV1(httpClient, sql, makeClientSettings());
+    }
+
+    DatabendClient startQuery(String sql, StageAttachment attach) throws SQLException {
         PaginationOptions options = getPaginationOptions();
         Map<String, String> additionalHeaders = setAdditionalHeaders();
-        additionalHeaders.put(X_Databend_Query_ID, UUID.randomUUID().toString());
         ClientSettings s = new ClientSettings.Builder().
+                setSession(this.session.get()).
+                setHost(this.getURI().toString()).
                 setQueryTimeoutSecs(this.driverUri.getQueryTimeout()).
                 setConnectionTimeout(this.driverUri.getConnectionTimeout()).
                 setSocketTimeout(this.driverUri.getSocketTimeout()).
-                setSession(this.session.get()).
-                setHost(this.getURI().toString()).
+                setPaginationOptions(options).
                 setAdditionalHeaders(additionalHeaders).
-                setPaginationOptions(options).build();
+                setStageAttachment(attach).
+                build();
         return new DatabendClientV1(httpClient, sql, s);
     }
 
-    DatabendClient startQuery(String sql, StageAttachment attach) throws SQLException {
+    private ClientSettings makeClientSettings() {
         PaginationOptions options = getPaginationOptions();
         Map<String, String> additionalHeaders = setAdditionalHeaders();
         ClientSettings s = new ClientSettings.Builder().
@@ -543,9 +588,8 @@ DatabendClient startQuery(String sql, StageAttachment attach) throws SQLExceptio
                 setSocketTimeout(this.driverUri.getSocketTimeout()).
                 setPaginationOptions(options).
                 setAdditionalHeaders(additionalHeaders).
-                setStageAttachment(attach).
                 build();
-        return new DatabendClientV1(httpClient, sql, s);
+        return s;
     }
 
     private Map<String, String> setAdditionalHeaders() {

databend-jdbc/src/main/java/com/databend/jdbc/DatabendDatabaseMetaData.java

@@ -992,7 +992,7 @@ private ResultSet getColumnsMetaDataBySQL(String sql) throws SQLException {
                 result.add(rowType.getDecimalDigits());// DECIMAL_DIGITS
                 result.add(0);// NUM_PREC_RADIX
                 result.add(rs.getString(6));// COLUMN_NAME
-                result.add(rs.getString(7));// REMARKS
+                result.add(rs.getObject(7));// REMARKS
                 result.add(rs.getString(8));// COLUMN_DEF
                 result.add(0);// SQL_DATA_TYPE
                 result.add(0);// SQL_DATETIME_SUB

databend-jdbc/src/main/java/com/databend/jdbc/DatabendDriverUri.java

@@ -40,6 +40,7 @@ public final class DatabendDriverUri {
     private final Properties properties;
     private final URI uri;
     private final boolean useSecureConnection;
+    private final boolean strNullAsNull;
     private final String warehouse;
     private final String sslmode;
     private final String tenant;
@@ -62,6 +63,7 @@ private DatabendDriverUri(String url, Properties driverProperties)
         Map.Entry<URI, Map<String, String>> uriAndProperties = parse(url);
         this.properties = mergeProperties(uriAndProperties.getKey(), uriAndProperties.getValue(), driverProperties);
         this.useSecureConnection = SSL.getValue(properties).orElse(false);
+        this.strNullAsNull = STRNULL_AS_NULL.getValue(properties).orElse(true);
         this.warehouse = WAREHOUSE.getValue(properties).orElse("");
         this.sslmode = SSL_MODE.getValue(properties).orElse("disable");
         this.tenant = TENANT.getValue(properties).orElse("");
@@ -258,6 +260,10 @@ public String getWarehouse() {
         return warehouse;
     }
 
+    public boolean getStrNullAsNull() {
+        return strNullAsNull;
+    }
+
     public String getSslmode() {
         return sslmode;
     }

databend-jdbc/src/main/java/com/databend/jdbc/NonRegisteringDatabendDriver.java

@@ -3,6 +3,7 @@
 import okhttp3.OkHttpClient;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.sql.Connection;
 import java.sql.Driver;
 import java.sql.DriverPropertyInfo;
@@ -67,6 +68,13 @@ public Connection connect(String url, Properties info)
 
         OkHttpClient.Builder builder = httpClient.newBuilder();
         uri.setupClient(builder);
+        DatabendConnection connection = new DatabendConnection(uri, builder.build());
+        // ping the server host
+        try {
+            connection.PingDatabendClientV1();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
 
         return new DatabendConnection(uri, builder.build());
     }

databend-jdbc/src/test/java/com/databend/jdbc/TestBasicDriver.java

@@ -39,6 +39,8 @@ public void setUp()
         c.createStatement().execute("create table test_basic_driver.table1(i int)");
         c.createStatement().execute("insert into test_basic_driver.table1 values(1)");
         c.createStatement().execute("create database test_basic_driver_2");
+        c.createStatement().execute("create table test_basic_driver.table_with_null(a int,b varchar default null)");
+        c.createStatement().execute("insert into test_basic_driver.table_with_null(a) values(1)");
 
         // json data
     }
@@ -65,6 +67,20 @@ public void testBasic()
         }
     }
 
+    @Test
+    public void testDefaultSelectNullValue() throws SQLException {
+        try (Connection connection = createConnection()) {
+            DatabendStatement statement = (DatabendStatement) connection.createStatement();
+            statement.executeQuery("SELECT a,b from test_basic_driver.table_with_null");
+            ResultSet r = statement.getResultSet();
+            r.next();
+            Assert.assertEquals(r.getInt(1), 1);
+            Assert.assertEquals(r.getObject(2), null);
+        } catch (SQLException throwables) {
+            throwables.printStackTrace();
+        }
+    }
+
     @Test(groups = {"IT"})
     public void testQueryUpdateCount()
             throws SQLException {

databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendDatabaseMetaData.java

@@ -62,8 +62,10 @@ public void setUp()
         Connection c = createConnection();
         c.createStatement().execute("drop table if exists test_column_meta");
         c.createStatement().execute("drop table if exists decimal_test");
+        c.createStatement().execute("drop table if exists test_comment");
         c.createStatement().execute("create table test_column_meta (nu1 uint8 null, u1 uint8, u2 uint16, u3 uint32, u4 uint64, i1 int8, i2 int16, i3 int32, i4 int64, f1 float32, f2 float64, s1 string,d1 date, d2 datetime, v1 variant, a1 array(int64), t1 Tuple(x Int64, y Int64 NULL)) engine = fuse");
         c.createStatement().execute("create table decimal_test (a decimal(4,2))");
+        c.createStatement().execute("create table test_comment (a int comment 'test comment')");
         // json data
     }
 
@@ -138,6 +140,23 @@ public void testGetColumns() throws Exception {
         }
     }
 
+    @Test(groups = {"IT"})
+    public void testComment() throws SQLException {
+        try (Connection connection = createConnection()) {
+            DatabaseMetaData metaData = connection.getMetaData();
+            try (ResultSet rs = connection.getMetaData().getColumns("default", "default", "test_comment", null)) {
+                while (rs.next()) {
+                    String tableSchem = rs.getString("table_schem");
+                    String tableName = rs.getString("table_name");
+                    String columnName = rs.getString("COLUMN_NAME");
+                    String remarks = rs.getString("remarks");
+                    Assert.assertEquals(remarks, "test comment");
+                    System.out.println(tableSchem + " " + tableName + " " + columnName + " " + remarks);
+                }
+            }
+        }
+    }
+
     @Test(groups = {"IT"})
     public void testColumnsMeta() throws Exception {
         try (Connection connection = createConnection()) {
@@ -159,6 +178,8 @@ public void testColumnsMeta() throws Exception {
                     String columnName = rs.getString("COLUMN_NAME");
                     int dataType = rs.getInt("data_type");
                     String columnType = rs.getString("type_name");
+                    Object remarks = rs.getObject("remarks");
+                    Assert.assertEquals(remarks, null);
                     System.out.println(tableSchem + " " + tableName + " " + columnName + " " + dataType + " " + columnType);
                 }
             }
@@ -176,6 +197,7 @@ public void testGetColumnTypesBySelectEmpty() throws Exception {
             }
         }
     }
+
     @Test(groups = {"IT"})
     public void testGetColumnTypeWithDecimal() throws Exception {
         try (Connection connection = createConnection()) {

databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendDriverUri.java

@@ -187,6 +187,7 @@ public void testFull() throws SQLException {
         props.setProperty("connection_time", "15");
         props.setProperty("warehouse", "test");
         props.setProperty("tenant", "tenant1");
+        props.setProperty("strnullasnull", String.valueOf(false));
         DatabendDriverUri uri = DatabendDriverUri.create("jdbc:databend://u1@localhost:33101/db1?password=p1&database=db2&tenant=tenant1&warehouse=test&null_display=null&connection_timeout=15&socket_timeout=15&presigned_url_disabled=true&wait_time_secs=1&max_rows_in_buffer=10&max_rows_per_page=5", props);
 
         Assert.assertEquals(uri.getProperties().get("user"), "u1");
@@ -204,5 +205,6 @@ public void testFull() throws SQLException {
         Assert.assertEquals(uri.getMaxRowsPerPage().intValue(), 7);
         Assert.assertFalse(uri.presignedUrlDisabled().booleanValue());
         Assert.assertEquals("null", uri.nullDisplay().toString());
+        Assert.assertEquals(false, uri.getStrNullAsNull());
     }
 }