From c8588f1c6dbd0e1d1bbaca08ff78c988d249104d Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 10 Jan 2024 16:00:54 +0800 Subject: [PATCH 1/9] feat: support parameter metadata --- .../com/databend/client/DatabendClientV1.java | 2 + databend-jdbc/pom.xml | 10 +- .../jdbc/DatabendParameterMetaData.java | 91 +++++++++++ .../jdbc/DatabendPreparedStatement.java | 2 + .../com/databend/jdbc/JdbcTypeMapping.java | 149 ++++++++++++++++++ .../jdbc/cloud/DatabendPresignClientV1.java | 8 +- 6 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java create mode 100644 databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java diff --git a/databend-client/src/main/java/com/databend/client/DatabendClientV1.java b/databend-client/src/main/java/com/databend/client/DatabendClientV1.java index 46160ead..2aeb67cf 100644 --- a/databend-client/src/main/java/com/databend/client/DatabendClientV1.java +++ b/databend-client/src/main/java/com/databend/client/DatabendClientV1.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; import static com.databend.client.JsonCodec.jsonCodec; import static com.google.common.base.MoreObjects.firstNonNull; @@ -60,6 +61,7 @@ public class DatabendClientV1 // client session private final AtomicReference databendSession; private final AtomicReference currentResults = new AtomicReference<>(); + private static final Logger logger = Logger.getLogger(DatabendClientV1.class.getPackage().getName()); public DatabendClientV1(OkHttpClient httpClient, String sql, ClientSettings settings) { requireNonNull(httpClient, "httpClient is null"); diff --git a/databend-jdbc/pom.xml b/databend-jdbc/pom.xml index 68734aaf..9b69ae04 100644 --- a/databend-jdbc/pom.xml +++ b/databend-jdbc/pom.xml @@ -49,11 +49,13 @@ org.slf4j slf4j-api + 1.7.6 - - org.slf4j - slf4j-simple - + + + + + com.google.guava guava diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java new file mode 100644 index 00000000..a9535312 --- /dev/null +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java @@ -0,0 +1,91 @@ +package com.databend.jdbc; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public class DatabendParameterMetaData implements ParameterMetaData { + protected final List params; + protected final JdbcTypeMapping mapper; + + protected DatabendParameterMetaData(List params, JdbcTypeMapping mapper) { + this.params = requireNonNull(params, "connection is null"); + this.mapper = mapper; + } + + protected DatabendColumnInfo getParameter(int param) throws SQLException { + if (param < 1 || param > params.size()) { + throw new RuntimeException(format("Parameter index should between 1 and %d but we got %d", params.size(), param)); + } + + return params.get(param - 1); + } + + public static String format(String template, Object... args) { + return String.format(Locale.ROOT, template, args); + } + + @Override + public int getParameterCount() throws SQLException { + return params.size(); + } + + @Override + public int isNullable(int param) throws SQLException { + DatabendColumnInfo p = getParameter(param); + if (p == null) { + return ParameterMetaData.parameterNullableUnknown; + } + + return p.getType().isNullable() ? ParameterMetaData.parameterNullable : ParameterMetaData.parameterNoNulls; + } + + @Override + public boolean isSigned(int param) throws SQLException { + return false; + } + + @Override + public int getPrecision(int param) throws SQLException { + return 0; + } + + @Override + public int getScale(int param) throws SQLException { + return 0; + } + + @Override + public int getParameterType(int param) throws SQLException { + return 0; + } + + @Override + public String getParameterTypeName(int param) throws SQLException { + return null; + } + + @Override + public String getParameterClassName(int param) throws SQLException { + return null; + } + + @Override + public int getParameterMode(int param) throws SQLException { + return 0; + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } +} diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java index 6f2b12e5..3d26d716 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java @@ -202,11 +202,13 @@ private StageAttachment uploadBatches() throws SQLException { LocalDateTime.now().getSecond(), uuid); String fileName = saved.getName(); + // upload to stage c.uploadStream(null, stagePrefix, fis, fileName, saved.length(), false); String stagePath = "@~/" + stagePrefix + fileName; Map copyOptions = new HashMap<>(); copyOptions.put("PURGE", String.valueOf(c.copyPurge())); copyOptions.put("NULL_DISPLAY", String.valueOf(c.nullDisplay())); + // insert with stage attachment StageAttachment attachment = new StageAttachment.Builder().setLocation(stagePath).setCopyOptions(copyOptions) .build(); return attachment; diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java new file mode 100644 index 00000000..8664d363 --- /dev/null +++ b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java @@ -0,0 +1,149 @@ +package com.databend.jdbc; + +import com.databend.client.data.DatabendDataType; + +import java.sql.Types; +import java.util.Map; + +public class JdbcTypeMapping { + /** + * Converts {@link DatabendColumnInfo} to generic SQL type defined in JDBC. + * + * @param column non-null column definition + * @return generic SQL type defined in JDBC + */ + public int toSqlType(DatabendColumnInfo column) { + DatabendDataType dataType = column.getType().getDataType(); + int sqlType = Types.OTHER; + switch (dataType) { + case BOOLEAN: + sqlType = Types.BOOLEAN; + break; + case INT_8: + sqlType = Types.TINYINT; + break; + case INT_16: + sqlType = Types.SMALLINT; + break; + case INT_32: + sqlType = Types.INTEGER; + break; + case INT_64: + sqlType = Types.BIGINT; + break; + case FLOAT: + sqlType = Types.FLOAT; + break; + case DOUBLE: + sqlType = Types.DOUBLE; + break; + case DECIMAL: + sqlType = Types.DECIMAL; + break; + case STRING: + sqlType = Types.VARCHAR; + break; + case DATE: + sqlType = Types.DATE; + break; + case TIMESTAMP: + sqlType = Types.TIMESTAMP; + break; + case ARRAY: + sqlType = Types.ARRAY; + break; + case VARIANT: + sqlType = Types.OTHER; + break; + case TUPLE: + sqlType = Types.STRUCT; + break; + case NULL: + sqlType = Types.NULL; + break; + default: + break; + } + return sqlType; + } + + /** + * Gets corresponding {@link DatabendDataType} of the given {@link Types}. + * + * @param sqlType generic SQL types defined in JDBC + * @return non-null Databend data type + */ + protected DatabendDataType getDataType(int sqlType) { + DatabendDataType dataType; + + switch (sqlType) { + case Types.BOOLEAN: + dataType = DatabendDataType.UNSIGNED_INT_8; + break; + case Types.TINYINT: + dataType = DatabendDataType.INT_8; + break; + case Types.SMALLINT: + dataType = DatabendDataType.INT_16; + break; + case Types.INTEGER: + dataType = DatabendDataType.INT_32; + break; + case Types.BIGINT: + dataType = DatabendDataType.INT_64; + break; + case Types.FLOAT: + dataType = DatabendDataType.FLOAT; + break; + case Types.DOUBLE: + dataType = DatabendDataType.DOUBLE; + break; + case Types.DECIMAL: + dataType = DatabendDataType.DECIMAL; + break; + case Types.BIT: + case Types.BLOB: + case Types.BINARY: + case Types.CHAR: + case Types.CLOB: + case Types.JAVA_OBJECT: + case Types.LONGNVARCHAR: + case Types.LONGVARBINARY: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NCLOB: + case Types.NVARCHAR: + case Types.OTHER: + case Types.SQLXML: + case Types.VARBINARY: + case Types.VARCHAR: + dataType = DatabendDataType.STRING; + break; + case Types.DATE: + dataType = DatabendDataType.DATE; + break; + case Types.TIME: + case Types.TIME_WITH_TIMEZONE: + case Types.TIMESTAMP: + case Types.TIMESTAMP_WITH_TIMEZONE: + dataType = DatabendDataType.TIMESTAMP; + break; + case Types.ARRAY: + dataType = DatabendDataType.ARRAY; + break; + case Types.STRUCT: + dataType = DatabendDataType.TUPLE; + break; + case Types.DATALINK: + case Types.DISTINCT: + case Types.REF: + case Types.REF_CURSOR: + case Types.ROWID: + case Types.NULL: + default: + dataType = DatabendDataType.NULL; + break; + } + return dataType; + } +} diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java index 2c8f05ad..00e1a49e 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java @@ -1,5 +1,6 @@ package com.databend.jdbc.cloud; +import com.databend.client.DatabendClientV1; import okhttp3.Headers; import okhttp3.HttpUrl; import okhttp3.MediaType; @@ -37,6 +38,7 @@ public class DatabendPresignClientV1 implements DatabendPresignClient { private static final Duration RetryTimeout = Duration.ofMinutes(5); private final OkHttpClient client; private final String uri; + private static final Logger logger = Logger.getLogger(DatabendPresignClientV1.class.getPackage().getName()); public DatabendPresignClientV1(OkHttpClient client, String uri) { Logger.getLogger(OkHttpClient.class.getName()).setLevel(Level.FINEST); @@ -100,13 +102,14 @@ private ResponseBody executeInternal(Request request, boolean shouldClose) throw Exception cause = null; while (true) { if (attempts > 0) { + logger.info("try to presign upload again: " + attempts); Duration sinceStart = Duration.ofNanos(System.nanoTime() - start); if (sinceStart.getSeconds() >= 60) { - System.out.println("Presign failed" + cause.toString()); + logger.warning("Presign upload failed, error is:" + cause.toString()); throw new RuntimeException(format("Error execute presign (attempts: %s, duration: %s)", attempts, sinceStart), cause); } if (attempts >= MaxRetryAttempts) { - System.out.println("Presign failed" + cause.toString()); + logger.warning("Presign upload failed, error is: " + cause.toString()); throw new RuntimeException(format("Error execute presign (attempts: %s, duration: %s)", attempts, sinceStart), cause); } @@ -127,7 +130,6 @@ private ResponseBody executeInternal(Request request, boolean shouldClose) throw if (response.isSuccessful()) { return response.body(); } else if (response.code() == 401) { - throw new RuntimeException("Error exeucte presign, Unauthorized user: " + response.code() + " " + response.message()); } else if (response.code() >= 503) { cause = new RuntimeException("Error execute presign, service unavailable: " + response.code() + " " + response.message()); From de7354f5db3bae6521f06fdab26534f2a734cf6b Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 10 Jan 2024 16:14:24 +0800 Subject: [PATCH 2/9] impl more method --- .../java/com/databend/jdbc/DatabendParameterMetaData.java | 3 ++- .../com/databend/jdbc/cloud/DatabendPresignClientV1.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java index a9535312..98ca8ecd 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java @@ -46,7 +46,8 @@ public int isNullable(int param) throws SQLException { @Override public boolean isSigned(int param) throws SQLException { - return false; + DatabendColumnInfo p = getParameter(param); + return p != null && p.isSigned(); } @Override diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java index 00e1a49e..adbc9c47 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java @@ -44,9 +44,9 @@ public DatabendPresignClientV1(OkHttpClient client, String uri) { Logger.getLogger(OkHttpClient.class.getName()).setLevel(Level.FINEST); OkHttpClient.Builder builder = client.newBuilder(); builder - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(90, TimeUnit.SECONDS) - .writeTimeout(90, TimeUnit.SECONDS); + .connectTimeout(300, TimeUnit.SECONDS) + .readTimeout(600, TimeUnit.SECONDS) + .writeTimeout(600, TimeUnit.SECONDS); this.client = client; this.uri = uri; } From 720f5ba9370c31813738e9ddc461cb877dd59e64 Mon Sep 17 00:00:00 2001 From: hantmac Date: Wed, 10 Jan 2024 17:30:55 +0800 Subject: [PATCH 3/9] more method --- .../jdbc/DatabendParameterMetaData.java | 27 ++++++++----------- .../java/com/databend/jdbc/JdbcWrapper.java | 22 +++++++++++++++ .../jdbc/cloud/DatabendPresignClientV1.java | 4 +++ 3 files changed, 37 insertions(+), 16 deletions(-) create mode 100644 databend-jdbc/src/main/java/com/databend/jdbc/JdbcWrapper.java diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java index 98ca8ecd..95e06207 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java @@ -2,13 +2,14 @@ import java.sql.ParameterMetaData; import java.sql.SQLException; +import java.sql.Types; import java.util.List; import java.util.Locale; import java.util.Map; import static java.util.Objects.requireNonNull; -public class DatabendParameterMetaData implements ParameterMetaData { +public class DatabendParameterMetaData extends JdbcWrapper implements ParameterMetaData { protected final List params; protected final JdbcTypeMapping mapper; @@ -52,22 +53,26 @@ public boolean isSigned(int param) throws SQLException { @Override public int getPrecision(int param) throws SQLException { - return 0; + DatabendColumnInfo p = getParameter(param); + return p != null ? p.getPrecision() : 0; } @Override public int getScale(int param) throws SQLException { - return 0; + DatabendColumnInfo p = getParameter(param); + return p != null ? p.getScale() : 0; } @Override public int getParameterType(int param) throws SQLException { - return 0; + DatabendColumnInfo p = getParameter(param); + return p != null ? mapper.toSqlType(p) : Types.OTHER; } @Override public String getParameterTypeName(int param) throws SQLException { - return null; + DatabendColumnInfo p = getParameter(param); + return p != null ? p.getColumnTypeName() : ""; } @Override @@ -77,16 +82,6 @@ public String getParameterClassName(int param) throws SQLException { @Override public int getParameterMode(int param) throws SQLException { - return 0; - } - - @Override - public T unwrap(Class iface) throws SQLException { - return null; - } - - @Override - public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return ParameterMetaData.parameterModeIn; } } diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/JdbcWrapper.java b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcWrapper.java new file mode 100644 index 00000000..70fb383b --- /dev/null +++ b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcWrapper.java @@ -0,0 +1,22 @@ +package com.databend.jdbc; + +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; + +public abstract class JdbcWrapper { + public T unwrap(Class iface) throws SQLException { + if (iface.isAssignableFrom(getClass())) { + return iface.cast(this); + } + + throw unsupportedError("Cannot unwrap to " + iface.getName()); + } + + public boolean isWrapperFor(Class iface) throws SQLException { + return iface.isAssignableFrom(getClass()); + } + + public static SQLFeatureNotSupportedException unsupportedError(String message) { + return new SQLFeatureNotSupportedException(message); + } +} diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java index adbc9c47..c8bb473d 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.SocketTimeoutException; import java.nio.file.Files; import java.time.Duration; import java.util.concurrent.TimeUnit; @@ -136,6 +137,9 @@ private ResponseBody executeInternal(Request request, boolean shouldClose) throw } else if (response.code() >= 400) { cause = new RuntimeException("Error execute presign, configuration error: " + response.code() + " " + response.message()); } + } catch (SocketTimeoutException e) { + logger.warning("Error execute presign, socket timeout: " + e.getMessage()); + cause = new RuntimeException("Error execute presign, socket timeout: " + e.getMessage()); } catch (RuntimeException e) { cause = e; } finally { From 019195a15822b5b96375ec8fa67dd175e70fd8ad Mon Sep 17 00:00:00 2001 From: hantmac Date: Thu, 11 Jan 2024 10:09:34 +0800 Subject: [PATCH 4/9] fix decimal type column --- .../src/main/java/com/databend/client/data/DatabendTypes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databend-client/src/main/java/com/databend/client/data/DatabendTypes.java b/databend-client/src/main/java/com/databend/client/data/DatabendTypes.java index 2093bc31..a203dd88 100644 --- a/databend-client/src/main/java/com/databend/client/data/DatabendTypes.java +++ b/databend-client/src/main/java/com/databend/client/data/DatabendTypes.java @@ -43,5 +43,5 @@ public final class DatabendTypes public static final String VARIANT_ARRAY = "variantarray"; public static final String VARIANT_OBJECT = "variantobject"; public static final String INTERVAL = "interval"; - public static final String DECIMAL = "Decimal"; + public static final String DECIMAL = "decimal"; } From 8b6bd6ad8d06d2aae0195d00005327a19a71ffc0 Mon Sep 17 00:00:00 2001 From: hantmac Date: Thu, 11 Jan 2024 10:45:18 +0800 Subject: [PATCH 5/9] fix client tiemout build --- .../java/com/databend/jdbc/cloud/DatabendPresignClientV1.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java index c8bb473d..a11038bd 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/cloud/DatabendPresignClientV1.java @@ -47,7 +47,7 @@ public DatabendPresignClientV1(OkHttpClient client, String uri) { builder .connectTimeout(300, TimeUnit.SECONDS) .readTimeout(600, TimeUnit.SECONDS) - .writeTimeout(600, TimeUnit.SECONDS); + .writeTimeout(600, TimeUnit.SECONDS).build(); this.client = client; this.uri = uri; } From b40bfd589198a409f51bf68a4a5a04b45669e4c3 Mon Sep 17 00:00:00 2001 From: hantmac Date: Mon, 15 Jan 2024 17:06:49 +0800 Subject: [PATCH 6/9] initialize the DatabendParameterMetaData --- .../client/data/DatabendDataType.java | 2 +- .../com/databend/jdbc/DatabendColumnInfo.java | 13 +++- .../jdbc/DatabendParameterMetaData.java | 4 +- .../jdbc/DatabendPreparedStatement.java | 14 ++++- .../jdbc/DatabendResultSetMetaData.java | 4 +- .../com/databend/jdbc/JdbcTypeMapping.java | 2 +- .../com/databend/jdbc/StatementUtilTest.java | 1 + .../jdbc/TestDatabendParameterMetaData.java | 59 +++++++++++++++++++ 8 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java diff --git a/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java b/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java index d3dd0d4b..5d301b0d 100644 --- a/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java +++ b/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java @@ -46,7 +46,7 @@ public enum DatabendDataType { MAP(Types.OTHER, DatabendTypes.MAP, false, 0, false, "Map"), BITMAP(Types.OTHER, DatabendTypes.MAP, false, 0, false, "Bitmap"), TUPLE(Types.OTHER, DatabendTypes.TUPLE, false, 0, false, "Tuple"), - VARIANT(Types.OTHER, DatabendTypes.VARIANT, false, 0, false, "Variant", "Json"), + VARIANT(Types.VARCHAR, DatabendTypes.VARIANT, false, 0, false, "Variant", "Json"), NULL(Types.NULL, DatabendTypes.NULL, false, 0, false, "NULL"), ; diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendColumnInfo.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendColumnInfo.java index 4e4f9940..e749eac3 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendColumnInfo.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendColumnInfo.java @@ -57,7 +57,9 @@ public DatabendColumnInfo(int columnType, List columnParameterTypes, Da public static DatabendColumnInfo of(String name, DatabendRawType type) { Preconditions.checkArgument(!Strings.isNullOrEmpty(name), "Provided name is null or empty"); - return newBuilder(name, type).build(); + Builder builder = newBuilder(name, type); + setTypeInfo(builder, type); + return builder.build(); } public static void setTypeInfo(Builder builder, DatabendRawType type) { @@ -163,7 +165,9 @@ public static void setTypeInfo(Builder builder, DatabendRawType type) { } public static Builder newBuilder(String name, DatabendRawType type) { - return (new Builder()).setColumnName(name).setColumnType(type.getDataType().getSqlType()); + return (new Builder()) + .setColumnName(name) + .setDatabendRawType(type); } public int getColumnType() { @@ -273,6 +277,11 @@ public Builder setColumnType(int columnType) { return this; } + public Builder setDatabendRawType(DatabendRawType type) { + this.type = type; + return this; + } + public void setColumnParameterTypes(List columnParameterTypes) { this.columnParameterTypes = ImmutableList.copyOf(requireNonNull(columnParameterTypes, "columnParameterTypes is null")); } diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java index 95e06207..c1d23199 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendParameterMetaData.java @@ -7,6 +7,7 @@ import java.util.Locale; import java.util.Map; +import static com.databend.jdbc.DatabendResultSetMetaData.getTypeClassName; import static java.util.Objects.requireNonNull; public class DatabendParameterMetaData extends JdbcWrapper implements ParameterMetaData { @@ -77,7 +78,8 @@ public String getParameterTypeName(int param) throws SQLException { @Override public String getParameterClassName(int param) throws SQLException { - return null; + DatabendColumnInfo p = getParameter(param); + return getTypeClassName(p.getColumnType()); } @Override diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java index 3d26d716..e3e24d93 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java @@ -1,6 +1,8 @@ package com.databend.jdbc; import com.databend.client.StageAttachment; +import com.databend.client.data.DatabendDataType; +import com.databend.client.data.DatabendRawType; import com.databend.jdbc.cloud.DatabendCopyParams; import com.databend.jdbc.cloud.DatabendStage; import com.databend.jdbc.parser.BatchInsertUtils; @@ -66,6 +68,7 @@ public class DatabendPreparedStatement extends DatabendStatement implements Prep private final RawStatementWrapper rawStatement; static final DateTimeFormatter TIME_FORMATTER = DateTimeFormat.forPattern("HH:mm:ss.SSS"); static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS"); + private final DatabendParameterMetaData paramMetaData; private static final java.time.format.DateTimeFormatter LOCAL_DATE_TIME_FORMATTER = new DateTimeFormatterBuilder() .append(ISO_LOCAL_DATE) @@ -90,6 +93,12 @@ public class DatabendPreparedStatement extends DatabendStatement implements Prep this.batchValues = new ArrayList<>(); this.batchInsertUtils = BatchInsertUtils.tryParseInsertSql(sql); this.rawStatement = StatementUtil.parseToRawStatementWrapper(sql); + int totalParams = (int) rawStatement.getTotalParams(); + List list = new ArrayList<>(totalParams); + for (int i = 1; i <= totalParams; i++) { + list.add(DatabendColumnInfo.of("parameter" + i, new DatabendRawType("VARIANT"))); + } + this.paramMetaData = new DatabendParameterMetaData(Collections.unmodifiableList(list), new JdbcTypeMapping()); } private static String formatBooleanLiteral(boolean x) { @@ -770,9 +779,8 @@ public void setURL(int i, URL url) } @Override - public ParameterMetaData getParameterMetaData() - throws SQLException { - return null; + public ParameterMetaData getParameterMetaData() throws SQLException { + return paramMetaData; } @Override diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendResultSetMetaData.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendResultSetMetaData.java index 8d492ea0..9dfe335e 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendResultSetMetaData.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendResultSetMetaData.java @@ -19,7 +19,7 @@ public class DatabendResultSetMetaData implements ResultSetMetaData { this.databendColumnInfo = databendColumnInfo; } - static String getType(int type) { + static String getTypeClassName(int type) { // see javax.sql.rowset.RowSetMetaDataImpl switch (type) { case Types.NUMERIC: @@ -198,7 +198,7 @@ public boolean isDefinitelyWritable(int i) @Override public String getColumnClassName(int i) throws SQLException { - return getType(column(i).getColumnType()); + return getTypeClassName(column(i).getColumnType()); } @Override diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java index 8664d363..09f0f052 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/JdbcTypeMapping.java @@ -53,7 +53,7 @@ public int toSqlType(DatabendColumnInfo column) { sqlType = Types.ARRAY; break; case VARIANT: - sqlType = Types.OTHER; + sqlType = Types.VARCHAR; break; case TUPLE: sqlType = Types.STRUCT; diff --git a/databend-jdbc/src/test/java/com/databend/jdbc/StatementUtilTest.java b/databend-jdbc/src/test/java/com/databend/jdbc/StatementUtilTest.java index d374e309..dc7af752 100644 --- a/databend-jdbc/src/test/java/com/databend/jdbc/StatementUtilTest.java +++ b/databend-jdbc/src/test/java/com/databend/jdbc/StatementUtilTest.java @@ -13,6 +13,7 @@ public class StatementUtilTest { void shouldGetAllQueryParamsFromIn() { String sql = "SElECT * FROM EMPLOYEES WHERE id IN (?,?)"; assertEquals(ImmutableMap.of(1, 37, 2, 39), StatementUtil.getParamMarketsPositions(sql)); + System.out.println(StatementUtil.parseToRawStatementWrapper(sql).getSubStatements()); assertEquals(1, StatementUtil.parseToRawStatementWrapper(sql).getSubStatements().size()); } @Test diff --git a/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java new file mode 100644 index 00000000..c950fcfa --- /dev/null +++ b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java @@ -0,0 +1,59 @@ +package com.databend.jdbc; + +import com.databend.client.data.DatabendDataType; +import org.testng.Assert; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import java.sql.*; +import java.util.Properties; + +public class TestDatabendParameterMetaData { + + private Connection createConnection() + throws SQLException { + String url = "jdbc:databend://localhost:8000"; + return DriverManager.getConnection(url, "databend", "databend"); + } + + private Connection createConnection(boolean presignDisabled) throws SQLException { + String url = "jdbc:databend://localhost:8000?presigned_url_disabled=" + presignDisabled; + return DriverManager.getConnection(url, "databend", "databend"); + } + + @BeforeTest + public void setUp() + throws SQLException { + // create table + Connection c = createConnection(); + System.out.println("-----------------"); + System.out.println("drop all existing test table"); + } + + + @Test(groups = "integration") + public void testGetParameterMetaData() throws SQLException { + try (Connection conn = createConnection(); + PreparedStatement emptyPs = conn.prepareStatement("select 1"); + // If you want to use ps.getParameterMetaData().* methods, you need to use a valid sql such as + // insert into table_name (col1 type1, col2 typ2, col3 type3) values (?, ?, ?) + PreparedStatement inputPs = conn.prepareStatement( + "insert into non_existing_table ('col2 String, col3 Int8, col1 VARIANT') values (?, ?, ?)"); + PreparedStatement sqlPs = conn.prepareStatement("select ?, toInt32(?), ? b");) { + Assert.assertEquals(emptyPs.getParameterMetaData().getParameterCount(), 0); + + for (PreparedStatement ps : new PreparedStatement[]{inputPs, sqlPs}) { + Assert.assertNotNull(ps.getParameterMetaData()); + Assert.assertTrue(ps.getParameterMetaData() == ps.getParameterMetaData(), + "parameter mete data should be singleton"); + Assert.assertEquals(ps.getParameterMetaData().getParameterCount(), 3); + Assert.assertEquals(ps.getParameterMetaData().getParameterMode(3), ParameterMetaData.parameterModeIn); + Assert.assertEquals(ps.getParameterMetaData().getParameterType(3), Types.VARCHAR); + Assert.assertEquals(ps.getParameterMetaData().getPrecision(3), 0); + Assert.assertEquals(ps.getParameterMetaData().getScale(3), 0); + Assert.assertEquals(ps.getParameterMetaData().getParameterClassName(3), String.class.getName()); + Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(3), DatabendDataType.VARIANT.name().toLowerCase()); + } + } + } +} From f9eafd199a46e133afeb97ad2ec479d595703fbc Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 16 Jan 2024 09:12:35 +0800 Subject: [PATCH 7/9] add test --- .../client/data/DatabendDataType.java | 2 +- .../jdbc/DatabendPreparedStatement.java | 30 +++++++++++++------ .../java/com/databend/jdbc/StatementUtil.java | 25 ++++++++++++++++ .../jdbc/TestDatabendParameterMetaData.java | 20 ++++++++++--- .../com/databend/jdbc/TestStatementUtil.java | 20 +++++++++++++ 5 files changed, 83 insertions(+), 14 deletions(-) create mode 100644 databend-jdbc/src/test/java/com/databend/jdbc/TestStatementUtil.java diff --git a/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java b/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java index 5d301b0d..d502f1ca 100644 --- a/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java +++ b/databend-client/src/main/java/com/databend/client/data/DatabendDataType.java @@ -85,7 +85,7 @@ public static DatabendDataType getByTypeName(String typeName) { return INT_16; } else if (DatabendTypes.UINT16.equalsIgnoreCase(typeName)) { return UNSIGNED_INT_16; - } else if (DatabendTypes.INT32.equalsIgnoreCase(typeName)) { + } else if (DatabendTypes.INT32.equalsIgnoreCase(typeName) || "int".equalsIgnoreCase(typeName)) { return INT_32; } else if (DatabendTypes.UINT32.equalsIgnoreCase(typeName)) { return UNSIGNED_INT_32; diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java index e3e24d93..e5594b20 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/DatabendPreparedStatement.java @@ -93,11 +93,12 @@ public class DatabendPreparedStatement extends DatabendStatement implements Prep this.batchValues = new ArrayList<>(); this.batchInsertUtils = BatchInsertUtils.tryParseInsertSql(sql); this.rawStatement = StatementUtil.parseToRawStatementWrapper(sql); - int totalParams = (int) rawStatement.getTotalParams(); - List list = new ArrayList<>(totalParams); - for (int i = 1; i <= totalParams; i++) { - list.add(DatabendColumnInfo.of("parameter" + i, new DatabendRawType("VARIANT"))); - } + Map params = StatementUtil.extractColumnTypes(sql); + List list = params.entrySet().stream().map(entry -> { + String type = entry.getValue(); + DatabendRawType databendRawType = new DatabendRawType(type); + return DatabendColumnInfo.of(entry.getKey().toString(), databendRawType); + }).collect(Collectors.toList()); this.paramMetaData = new DatabendParameterMetaData(Collections.unmodifiableList(list), new JdbcTypeMapping()); } @@ -725,15 +726,23 @@ public void setRef(int i, Ref ref) } @Override - public void setBlob(int i, Blob blob) + public void setBlob(int i, Blob x) throws SQLException { - throw new SQLFeatureNotSupportedException("PreparedStatement", "setBlob"); + if (x != null) { + setBinaryStream(i, x.getBinaryStream()); + } else { + setNull(i, Types.BLOB); + } } @Override - public void setClob(int i, Clob clob) + public void setClob(int i, Clob x) throws SQLException { - throw new SQLFeatureNotSupportedException("PreparedStatement", "setClob"); + if (x != null) { + setCharacterStream(i, x.getCharacterStream()); + } else { + setNull(i, Types.CLOB); + } } @Override @@ -778,6 +787,8 @@ public void setURL(int i, URL url) throw new SQLFeatureNotSupportedException("PreparedStatement", "setURL"); } + // If you want to use ps.getParameterMetaData().* methods, you need to use a valid sql such as + // insert into table_name (col1 type1, col2 typ2, col3 type3) values (?, ?, ?) @Override public ParameterMetaData getParameterMetaData() throws SQLException { return paramMetaData; @@ -897,6 +908,7 @@ public void setNClob(int i, Reader reader) throw new SQLFeatureNotSupportedException("PreparedStatement", "setNClob"); } + private String toDateLiteral(Object value) throws IllegalArgumentException { requireNonNull(value, "value is null"); if (value instanceof java.util.Date) { diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/StatementUtil.java b/databend-jdbc/src/main/java/com/databend/jdbc/StatementUtil.java index 9d0052f2..a4680ba7 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/StatementUtil.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/StatementUtil.java @@ -1,6 +1,7 @@ package com.databend.jdbc; import java.util.*; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -51,6 +52,30 @@ public Optional> extractParamFromSetStatement(@NonNull Stri return Optional.empty(); } + /** + * This method is used to extract column types from a SQL statement. + * It parses the SQL statement and finds the column types defined in the first pair of parentheses. + * The column types are then stored in a Map where the key is the index of the column in the SQL statement + * and the value is the type of the column. + * + * @param sql The SQL statement from which to extract column types. + * @return A Map where the key is the index of the column and the value is the type of the column. + */ + public static Map extractColumnTypes(String sql) { + Map columnTypes = new LinkedHashMap<>(); + Pattern pattern = Pattern.compile("\\((.*?)\\)"); + Matcher matcher = pattern.matcher(sql); + if (matcher.find()) { + String[] columns = matcher.group(1).split(","); + for (int i = 0; i < columns.length; i++) { + String column = columns[i].trim(); + String type = column.substring(column.lastIndexOf(' ') + 1).replace("'", ""); + columnTypes.put(i, type); + } + } + return columnTypes; + } + /** * Parse the sql statement to a list of {@link StatementInfoWrapper} * diff --git a/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java index c950fcfa..e85d0f96 100644 --- a/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java +++ b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java @@ -8,6 +8,7 @@ import java.sql.*; import java.util.Properties; + public class TestDatabendParameterMetaData { private Connection createConnection() @@ -38,8 +39,8 @@ public void testGetParameterMetaData() throws SQLException { // If you want to use ps.getParameterMetaData().* methods, you need to use a valid sql such as // insert into table_name (col1 type1, col2 typ2, col3 type3) values (?, ?, ?) PreparedStatement inputPs = conn.prepareStatement( - "insert into non_existing_table ('col2 String, col3 Int8, col1 VARIANT') values (?, ?, ?)"); - PreparedStatement sqlPs = conn.prepareStatement("select ?, toInt32(?), ? b");) { + "insert into non_existing_table ('col2 String, col3 Int8, col1 String') values (?, ?, ?)"); + PreparedStatement sqlPs = conn.prepareStatement("insert into test_table (a int, b int, c string) values (?,?,?)");) { Assert.assertEquals(emptyPs.getParameterMetaData().getParameterCount(), 0); for (PreparedStatement ps : new PreparedStatement[]{inputPs, sqlPs}) { @@ -49,11 +50,22 @@ public void testGetParameterMetaData() throws SQLException { Assert.assertEquals(ps.getParameterMetaData().getParameterCount(), 3); Assert.assertEquals(ps.getParameterMetaData().getParameterMode(3), ParameterMetaData.parameterModeIn); Assert.assertEquals(ps.getParameterMetaData().getParameterType(3), Types.VARCHAR); - Assert.assertEquals(ps.getParameterMetaData().getPrecision(3), 0); + Assert.assertEquals(ps.getParameterMetaData().getPrecision(3), 1024 * 1024 * 1024); Assert.assertEquals(ps.getParameterMetaData().getScale(3), 0); Assert.assertEquals(ps.getParameterMetaData().getParameterClassName(3), String.class.getName()); - Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(3), DatabendDataType.VARIANT.name().toLowerCase()); + Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(3), DatabendDataType.STRING.name().toLowerCase()); } } + + try (Connection conn = createConnection(); + PreparedStatement ps = conn.prepareStatement("insert into test_table (a int, b int) values (?,?)");) { + Assert.assertEquals(ps.getParameterMetaData().getParameterCount(), 2); + Assert.assertEquals(ps.getParameterMetaData().getParameterMode(2), ParameterMetaData.parameterModeIn); + Assert.assertEquals(ps.getParameterMetaData().getParameterType(2), Types.INTEGER); + Assert.assertEquals(ps.getParameterMetaData().getPrecision(2), 10); + Assert.assertEquals(ps.getParameterMetaData().getScale(2), 0); + Assert.assertEquals(ps.getParameterMetaData().getParameterClassName(2), Integer.class.getName()); + Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(2), DatabendDataType.INT_32.getDisplayName().toLowerCase()); + } } } diff --git a/databend-jdbc/src/test/java/com/databend/jdbc/TestStatementUtil.java b/databend-jdbc/src/test/java/com/databend/jdbc/TestStatementUtil.java new file mode 100644 index 00000000..6304bee9 --- /dev/null +++ b/databend-jdbc/src/test/java/com/databend/jdbc/TestStatementUtil.java @@ -0,0 +1,20 @@ +package com.databend.jdbc; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStatementUtil { + @Test + public void testExtractColumnTypes() { + String sql = "insert into non_existing_table ('col2 String, col3 Int8, col1 VARIANT') values (?, ?, ?)"; + Map columnTypes = StatementUtil.extractColumnTypes(sql); + + assertEquals(3, columnTypes.size()); + assertEquals("String", columnTypes.get(0)); + assertEquals("Int8", columnTypes.get(1)); + assertEquals("VARIANT", columnTypes.get(2)); + } +} From 043649eced7ebb620b2642f9547bbb25f4cf1774 Mon Sep 17 00:00:00 2001 From: hantmac Date: Tue, 16 Jan 2024 15:40:37 +0800 Subject: [PATCH 8/9] add tests --- .../databend/jdbc/TestDatabendParameterMetaData.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java index e85d0f96..e241c062 100644 --- a/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java +++ b/databend-jdbc/src/test/java/com/databend/jdbc/TestDatabendParameterMetaData.java @@ -67,5 +67,16 @@ public void testGetParameterMetaData() throws SQLException { Assert.assertEquals(ps.getParameterMetaData().getParameterClassName(2), Integer.class.getName()); Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(2), DatabendDataType.INT_32.getDisplayName().toLowerCase()); } + + try (Connection conn = createConnection(); + PreparedStatement ps = conn.prepareStatement("insert into test_table (a int, b VARIANT) values (?,?)");) { + Assert.assertEquals(ps.getParameterMetaData().getParameterCount(), 2); + Assert.assertEquals(ps.getParameterMetaData().getParameterMode(2), ParameterMetaData.parameterModeIn); + Assert.assertEquals(ps.getParameterMetaData().getParameterType(2), Types.VARCHAR); + Assert.assertEquals(ps.getParameterMetaData().getPrecision(2), 0); + Assert.assertEquals(ps.getParameterMetaData().getScale(2), 0); + Assert.assertEquals(ps.getParameterMetaData().getParameterClassName(2), String.class.getName()); + Assert.assertEquals(ps.getParameterMetaData().getParameterTypeName(2), DatabendDataType.VARIANT.getDisplayName().toLowerCase()); + } } } From 1b60aa5a26075089897dc00f994c70c35ce0f2f1 Mon Sep 17 00:00:00 2001 From: hantmac Date: Thu, 18 Jan 2024 10:09:53 +0800 Subject: [PATCH 9/9] fix exampl --- .../src/main/java/com/databend/jdbc/examples/Examples.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/databend-jdbc/src/main/java/com/databend/jdbc/examples/Examples.java b/databend-jdbc/src/main/java/com/databend/jdbc/examples/Examples.java index 2f56f401..f23c87f8 100644 --- a/databend-jdbc/src/main/java/com/databend/jdbc/examples/Examples.java +++ b/databend-jdbc/src/main/java/com/databend/jdbc/examples/Examples.java @@ -43,7 +43,7 @@ public static void main(String[] args) throws SQLException { try (PreparedStatement statement = conn.prepareStatement(updateSQL)) { statement.setInt(2, 1); // Attention: now setString(1, "c") will throw exception, need to setString(1, "'c'") - statement.setString(1, "'c'"); + statement.setString(1, "c"); int result = statement.executeUpdate(); System.out.println(result); }