Skip to content

Commit

Permalink
Merge pull request #1826 from ClickHouse/clientv2_primitive_reader
Browse files Browse the repository at this point in the history
[client-v2] primitive reading into pojo
  • Loading branch information
chernser authored Oct 1, 2024
2 parents 27e2613 + 0de9eb2 commit 18a6550
Show file tree
Hide file tree
Showing 7 changed files with 526 additions and 351 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,17 +76,27 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer

protected AtomicBoolean nextRecordEmpty = new AtomicBoolean(true);

/**
* Reads next record into POJO object using set of serializers.
* There should be a serializer for each column in the record, otherwise it will silently skip a field
* It is done in such a way because it is not the reader concern. Calling code should validate this.
*
* Note: internal API
* @param deserializers
* @param obj
* @return
* @throws IOException
*/
public boolean readToPOJO(Map<String, POJOSetter> deserializers, Object obj ) throws IOException {
boolean firstColumn = true;

for (ClickHouseColumn column : columns) {
try {
Object val = binaryStreamReader.readValue(column);
if (val != null) {
POJOSetter deserializer = deserializers.get(column.getColumnName());
if (deserializer != null) {
deserializer.setValue(obj, val);
}
POJOSetter deserializer = deserializers.get(column.getColumnName());
if (deserializer != null) {
deserializer.setValue(obj, binaryStreamReader, column);
} else {
binaryStreamReader.skipValue(column);
}
firstColumn = false;
} catch (EOFException e) {
Expand Down

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.clickhouse.client.api.query;


import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
import com.clickhouse.data.ClickHouseColumn;

/**
* Class used to set value for individual fields in a POJO.
* Implementation will have reference to a specific POJO property.
Expand All @@ -9,37 +12,5 @@
*/
public interface POJOSetter {

default void setValue(Object obj, boolean value) {
throw new UnsupportedOperationException("Unsupported type: boolean");
};

default void setValue(Object obj, byte value) {
throw new UnsupportedOperationException("Unsupported type: byte");
};

default void setValue(Object obj, char value) {
throw new UnsupportedOperationException("Unsupported type: char");
};

default void setValue(Object obj, short value) {
throw new UnsupportedOperationException("Unsupported type: short");
};

default void setValue(Object obj, int value) {
throw new UnsupportedOperationException("Unsupported type: int");
};

default void setValue(Object obj, long value) {
throw new UnsupportedOperationException("Unsupported type: long");
};

default void setValue(Object obj, float value) {
throw new UnsupportedOperationException("Unsupported type: float");
};

default void setValue(Object obj, double value) {
throw new UnsupportedOperationException("Unsupported type: double");
};

void setValue(Object obj, Object value);
void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws Exception;
}
Original file line number Diff line number Diff line change
@@ -1,62 +1,28 @@
package com.clickhouse.client.internal;

import com.clickhouse.client.api.internal.SerializerUtils;
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
import com.clickhouse.client.api.query.POJOSetter;
import com.clickhouse.client.query.SamplePOJO;
import com.clickhouse.client.query.QuerySamplePOJO;
import com.clickhouse.client.query.SimplePOJO;
import com.clickhouse.data.ClickHouseColumn;
import org.testng.Assert;
import org.testng.annotations.Test;

import java.lang.reflect.Method;
import java.math.BigInteger;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import java.time.ZonedDateTime;

public class SerializerUtilsTests {

@Test(enabled = false)
public void testDeserialize() throws Exception {

Map<String, POJOSetter> pojoSetterList = new HashMap<>();
for (Method method : SamplePOJOForSerialization.class.getDeclaredMethods()) {
if (method.getName().startsWith("set")) {
pojoSetterList.put(method.getName().substring(3).toLowerCase(),
SerializerUtils.compilePOJOSetter(method, ClickHouseColumn.of(method.getName(),
"String")));
}
}

SamplePOJOForSerialization pojo = new SamplePOJOForSerialization();
pojoSetterList.get("string").setValue(pojo, "John Doe");
pojoSetterList.get("int32").setValue(pojo, Integer.valueOf(30));
pojoSetterList.get("int16").setValue(pojo, 22);

Assert.assertEquals(pojo.getString(), "John Doe");
Assert.assertEquals(pojo.getInt32(), 30);
Assert.assertEquals(pojo.getInt16(), 22);
}

public static class SamplePOJOInt256Setter implements POJOSetter {




/*
public void setValue(java.lang.Object, java.lang.Object);
Code:
0: aload_1
1: checkcast #7 // class com/clickhouse/client/query/SamplePOJO
4: aload_2
5: checkcast #25 // class java/math/BigInteger
8: invokevirtual #27 // Method com/clickhouse/client/query/SamplePOJO.setInt256:(Ljava/math/BigInteger;)V
11: return
*/
@Override
public void setValue(Object obj, Object value) {
Arrays.stream(((Object[]) value)).collect(Collectors.toList());
public void setValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
((QuerySamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
}

public void readValue(Object obj, BinaryStreamReader reader, ClickHouseColumn column) throws IOException {
// ((SamplePOJO)obj).setDateTime(((ZonedDateTime)reader.readValue(column)).toLocalDateTime());
((SimplePOJO)obj).setId(reader.readIntLE());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import java.util.Random;
import java.util.UUID;

public class SamplePOJO {
public class QuerySamplePOJO {
private int int8;
private int int8_default;
private int int16;
Expand All @@ -35,7 +35,7 @@ public class SamplePOJO {
private int uint8;
private int uint16;
private long uint32;
private long uint64;
private BigInteger uint64;
private BigInteger uint128;
private BigInteger uint256;

Expand Down Expand Up @@ -67,12 +67,15 @@ public class SamplePOJO {
private Inet6Address ipv6;

private List<String> array;
private List<Integer> tuple;
private List<?> tuple;

private Object[] tupleArray;

private Map<String, Integer> map;
private List<Integer> nestedInnerInt;
private List<String> nestedInnerString;

public SamplePOJO() {
public QuerySamplePOJO() {
final Random random = new Random();
int8 = random.nextInt(128);
int16 = random.nextInt(32768);
Expand All @@ -90,11 +93,16 @@ public SamplePOJO() {

int256 = upper1.or(upper2).or(lower1).or(lower2);


uint8 = random.nextInt(255);
uint16 = random.nextInt(32768);
uint32 = (long) (random.nextDouble() * 4294967295L);
uint64 = (long) (random.nextDouble() * 18446744073709615L);

long rndUInt64 = random.nextLong();
uint64 = BigInteger.valueOf(rndUInt64);
if (rndUInt64 < 0) {
uint64 = uint64.add(BigInteger.ONE.shiftLeft(64));
}

uint128 = upper.or(lower).abs();
uint256 = upper1.or(upper2).or(lower1).or(lower2).abs();
Expand Down Expand Up @@ -137,7 +145,8 @@ public SamplePOJO() {
}

array = Arrays.asList("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z");
tuple = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
tuple = Arrays.asList(random.nextInt(), random.nextDouble(), "a", "b");
tupleArray = new Object[] {random.nextInt(), random.nextDouble(), "c", "d" };
map = new HashMap<>();
for (int i = 0; i < 10; i++) {
map.put(String.valueOf((char) ('a' + i)), i + 1);
Expand Down Expand Up @@ -248,11 +257,11 @@ public void setUint32(long uint32) {
this.uint32 = uint32;
}

public long getUint64() {
public BigInteger getUint64() {
return uint64;
}

public void setUint64(long uint64) {
public void setUint64(BigInteger uint64) {
this.uint64 = uint64;
}

Expand Down Expand Up @@ -424,14 +433,22 @@ public void setArray(List<String> array) {
this.array = array;
}

public List<Integer> getTuple() {
public List<?> getTuple() {
return tuple;
}

public void setTuple(List<Integer> tuple) {
public void setTuple(List<?> tuple) {
this.tuple = tuple;
}

public Object[] getTupleArray() {
return tupleArray;
}

public void setTupleArray(Object[] tupleArray) {
this.tupleArray = tupleArray;
}

public Map<String, Integer> getMap() {
return map;
}
Expand Down Expand Up @@ -460,18 +477,18 @@ public void setNestedInnerString(List<String> nestedInnerString) {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SamplePOJO that = (SamplePOJO) o;
return int8 == that.int8 && int16 == that.int16 && int32 == that.int32 && int64 == that.int64 && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && uint64 == that.uint64 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int256, that.int256) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(tuple, that.tuple) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
QuerySamplePOJO that = (QuerySamplePOJO) o;
return int8 == that.int8 && int8_default == that.int8_default && int16 == that.int16 && int16_default == that.int16_default && int32 == that.int32 && int32_default == that.int32_default && int64 == that.int64 && int64_default == that.int64_default && uint8 == that.uint8 && uint16 == that.uint16 && uint32 == that.uint32 && Float.compare(float32, that.float32) == 0 && Double.compare(float64, that.float64) == 0 && bool == that.bool && enum8 == that.enum8 && enum16 == that.enum16 && Objects.equals(int128, that.int128) && Objects.equals(int128_default, that.int128_default) && Objects.equals(int256, that.int256) && Objects.equals(int256_default, that.int256_default) && Objects.equals(uint64, that.uint64) && Objects.equals(uint128, that.uint128) && Objects.equals(uint256, that.uint256) && Objects.equals(decimal32, that.decimal32) && Objects.equals(decimal64, that.decimal64) && Objects.equals(decimal128, that.decimal128) && Objects.equals(decimal256, that.decimal256) && Objects.equals(string, that.string) && Objects.equals(fixedString, that.fixedString) && Objects.equals(date, that.date) && Objects.equals(date32, that.date32) && Objects.equals(dateTime, that.dateTime) && Objects.equals(dateTime64, that.dateTime64) && Objects.equals(uuid, that.uuid) && Objects.equals(ipv4, that.ipv4) && Objects.equals(ipv6, that.ipv6) && Objects.equals(array, that.array) && Objects.equals(map, that.map) && Objects.equals(nestedInnerInt, that.nestedInnerInt) && Objects.equals(nestedInnerString, that.nestedInnerString);
}

@Override
public int hashCode() {
return Objects.hash(int8, int16, int32, int64, int128, int256, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, tuple, map, nestedInnerInt, nestedInnerString);
return Objects.hash(int8, int8_default, int16, int16_default, int32, int32_default, int64, int64_default, int128, int128_default, int256, int256_default, uint8, uint16, uint32, uint64, uint128, uint256, float32, float64, decimal32, decimal64, decimal128, decimal256, bool, string, fixedString, date, date32, dateTime, dateTime64, uuid, enum8, enum16, ipv4, ipv6, array, map, nestedInnerInt, nestedInnerString);
}

@Override
public String toString() {
return "SamplePOJO{" +
return "QuerySamplePOJO{" +
"int8=" + int8 +
", int8_default=" + int8_default +
", int16=" + int16 +
Expand Down Expand Up @@ -509,7 +526,6 @@ public String toString() {
", ipv4=" + ipv4 +
", ipv6=" + ipv6 +
", array=" + array +
", tuple=" + tuple +
", map=" + map +
", nestedInnerInt=" + nestedInnerInt +
", nestedInnerString=" + nestedInnerString +
Expand Down Expand Up @@ -555,7 +571,8 @@ public static String generateTableCreateSQL(String tableName) {
"ipv4 IPv4, " +
"ipv6 IPv6, " +
"array Array(String), " +
"tuple Tuple(Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32, Int32), " +
"tuple Tuple(Int32, Float64, String, String), " +
"tupleArray Tuple(Int32, Float64, String, String), " +
"map Map(String, Int32), " +
"nested Nested (innerInt Int32, innerString String)" +
") ENGINE = Memory";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.testcontainers.shaded.com.google.common.collect.Table;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
Expand All @@ -47,7 +46,6 @@
import java.io.OutputStreamWriter;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
Expand Down Expand Up @@ -1542,13 +1540,13 @@ public void testQueryReadToPOJOWithoutGetters() {
public void testQueryAllWithPOJO() throws Exception {

final String tableName = "test_query_all_with_pojo";
final String createTableSQL = SamplePOJO.generateTableCreateSQL(tableName);
final String createTableSQL = QuerySamplePOJO.generateTableCreateSQL(tableName);
client.execute("DROP TABLE IF EXISTS test_query_all_with_pojo").get();
client.execute(createTableSQL).get();

SamplePOJO pojo = new SamplePOJO();
QuerySamplePOJO pojo = new QuerySamplePOJO();
TableSchema schema = client.getTableSchema(tableName);
client.register(SamplePOJO.class, schema);
client.register(QuerySamplePOJO.class, schema);

client.insert(tableName, Collections.singletonList(pojo)).get();

Expand All @@ -1562,7 +1560,7 @@ public void testQueryAllWithPOJO() throws Exception {
pojo.setDateTime(pojo.getDateTime().minusNanos(pojo.getDateTime().getNano()));
pojo.setDateTime64(pojo.getDateTime64().withNano((int) Math.ceil((pojo.getDateTime64().getNano() / 1000_000) * 1000_000)));

List<SamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", SamplePOJO.class,
List<QuerySamplePOJO> pojos = client.queryAll("SELECT * FROM " + tableName + " LIMIT 1", QuerySamplePOJO.class,
schema);
Assert.assertEquals(pojos.get(0), pojo, "Expected " + pojo + " but got " + pojos.get(0));
}
Expand Down

0 comments on commit 18a6550

Please sign in to comment.