From a9b46909ad6b16c9ef0c906f0a94d9dc3326bee0 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Sat, 14 Sep 2024 11:39:08 +0900 Subject: [PATCH] Add DuckDB connector --- .github/workflows/ci.yml | 2 + core/trino-server/src/main/provisio/trino.xml | 6 + docs/src/main/sphinx/connector.md | 1 + docs/src/main/sphinx/connector/duckdb.md | 207 +++++++++++ docs/src/main/sphinx/static/img/duckdb.png | Bin 0 -> 12403 bytes plugin/trino-duckdb/pom.xml | 206 +++++++++++ .../io/trino/plugin/duckdb/DuckDbClient.java | 300 ++++++++++++++++ .../io/trino/plugin/duckdb/DuckDbPlugin.java | 25 ++ .../plugin/duckdb/DuckDblClientModule.java | 63 ++++ .../plugin/duckdb/DuckDbQueryRunner.java | 108 ++++++ .../duckdb/TestDuckDbConnectorTest.java | 294 ++++++++++++++++ .../trino/plugin/duckdb/TestDuckDbPlugin.java | 37 ++ .../plugin/duckdb/TestDuckDbTypeMapping.java | 331 ++++++++++++++++++ .../io/trino/plugin/duckdb/TestingDuckDb.java | 71 ++++ pom.xml | 1 + .../EnvMultinodeAllConnectors.java | 1 + .../multinode-all/duckdb.properties | 2 + 17 files changed, 1655 insertions(+) create mode 100644 docs/src/main/sphinx/connector/duckdb.md create mode 100644 docs/src/main/sphinx/static/img/duckdb.png create mode 100644 plugin/trino-duckdb/pom.xml create mode 100644 plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbClient.java create mode 100644 plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbPlugin.java create mode 100644 plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDblClientModule.java create mode 100644 plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/DuckDbQueryRunner.java create mode 100644 plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbConnectorTest.java create mode 100644 plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbPlugin.java create mode 100644 plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbTypeMapping.java create mode 100644 plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestingDuckDb.java create mode 100644 testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-all/duckdb.properties diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd43742daaec8c..443f8f976f4813 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -341,6 +341,7 @@ jobs: !:trino-delta-lake, !:trino-docs, !:trino-druid, + !:trino-duckdb, !:trino-elasticsearch, !:trino-exasol, !:trino-faulttolerant-tests, @@ -453,6 +454,7 @@ jobs: - { modules: plugin/trino-delta-lake, profile: cloud-tests } - { modules: plugin/trino-delta-lake, profile: fte-tests } - { modules: plugin/trino-druid } + - { modules: plugin/trino-duckdb } - { modules: plugin/trino-elasticsearch } - { modules: plugin/trino-exasol } - { modules: plugin/trino-google-sheets } diff --git a/core/trino-server/src/main/provisio/trino.xml b/core/trino-server/src/main/provisio/trino.xml index 3844a44d61c9a6..d70716e961c53d 100644 --- a/core/trino-server/src/main/provisio/trino.xml +++ b/core/trino-server/src/main/provisio/trino.xml @@ -84,6 +84,12 @@ + + + + + + diff --git a/docs/src/main/sphinx/connector.md b/docs/src/main/sphinx/connector.md index 5dc77011477dbb..7396e69525b5cf 100644 --- a/docs/src/main/sphinx/connector.md +++ b/docs/src/main/sphinx/connector.md @@ -14,6 +14,7 @@ Cassandra ClickHouse Delta Lake Druid +DuckDB Elasticsearch Exasol Google Sheets diff --git a/docs/src/main/sphinx/connector/duckdb.md b/docs/src/main/sphinx/connector/duckdb.md new file mode 100644 index 00000000000000..69af660f1b28ec --- /dev/null +++ b/docs/src/main/sphinx/connector/duckdb.md @@ -0,0 +1,207 @@ +# DuckDB connector + +```{raw} html + +``` + +The DuckDB connector allows querying and creating tables in an external +[DuckDB](https://duckdb.org/) instance. This can be used to join data between +different systems like DuckDB and Hive, or between two different +DuckDB instances. + +## Configuration + +To configure the DuckDB connector, create a catalog properties file +in `etc/catalog` named, for example, `example.properties`, to +mount the DuckDB connector as the `duckdb` catalog. +Create the file with the following contents, replacing the +connection properties as appropriate for your setup: + +```none +connector.name=duckdb +connection-url=jdbc:duckdb:// +connection-user=root +connection-password=secret +``` + +### Multiple DuckDB servers + +The DuckDB connector can only access a single database within +a DuckDB instance. Thus, if you have multiple DuckDB servers, +or want to connect to multiple DuckDB servers, you must configure +multiple instances of the DuckDB connector. + +(duckdb-type-mapping)= +## Type mapping + +Because Trino and DuckDB each support types that the other does not, this +connector {ref}`modifies some types ` when reading or +writing data. Data types may not map the same way in both directions between +Trino and the data source. Refer to the following sections for type mapping in +each direction. + +List of [DuckDB data types](https://duckdb.org/docs/sql/data_types/overview.html). + +### DuckDB type to Trino type mapping + +The connector maps DuckDB types to the corresponding Trino types following +this table: + +:::{list-table} DuckDB type to Trino type mapping +:widths: 30, 30, 40 +:header-rows: 1 + +* - DuckDB type + - Trino type + - Notes +* - `BOOLEAN` + - `BOOLEAN` + - +* - `TINYINT` + - `TINYINT` + - +* - `SMALLINT` + - `SMALLINT` + - +* - `INTEGER` + - `INTEGER` + - +* - `BIGINT` + - `BIGINT` + - +* - `FLOAT` + - `REAL` + - +* - `DOUBLE` + - `DOUBLE` + - +* - `DECIMAL` + - `DECIMAL` + - Default precision and scale are (18,3). +* - `VARCHAR` + - `VARCHAR` + - +* - `DATE` + - `DATE` + - +::: + +No other types are supported. + +### Trino type to DuckDB type mapping + +The connector maps Trino types to the corresponding DuckDB types following +this table: + +:::{list-table} Trino type to DuckDB type mapping +:widths: 30, 30, 40 +:header-rows: 1 + +* - Trino type + - DuckDB type + - Notes +* - `BOOLEAN` + - `BOOLEAN` + - +* - `TINYINT` + - `TINYINT` + - +* - `SMALLINT` + - `SMALLINT` + - +* - `INTEGER` + - `INTEGER` + - +* - `BIGINT` + - `BIGINT` + - +* - `REAL` + - `REAL` + - +* - `DOUBLE` + - `DOUBLE` + - +* - `DECIMAL` + - `DECIMAL` + - +* - `CHAR` + - `VARCHAR` + - +* - `VARCHAR` + - `VARCHAR` + - +* - `DATE` + - `DATE` + - +::: + +No other types are supported. + +```{include} jdbc-type-mapping.fragment +``` + +(duckdb-sql-support)= +## SQL support + +The connector provides read access and write access to data and metadata in +a DuckDB database. In addition to the {ref}`globally available +` and {ref}`read operation ` +statements, the connector supports the following features: + +- {doc}`/sql/insert` +- {doc}`/sql/delete` +- {doc}`/sql/truncate` +- {doc}`/sql/create-table` +- {doc}`/sql/create-table-as` +- {doc}`/sql/drop-table` +- {doc}`/sql/alter-table` +- {doc}`/sql/create-schema` +- {doc}`/sql/drop-schema` + +### Procedures + +```{include} jdbc-procedures-flush.fragment +``` +```{include} procedures-execute.fragment +``` + +### Table functions + +The connector provides specific [table functions](/functions/table) to +access DuckDB. + +(duckdb-query-function)= +#### `query(varchar) -> table` + +The `query` function allows you to query the underlying database directly. It +requires syntax native to DuckDB, because the full query is pushed down and +processed in DuckDB. This can be useful for accessing native features which +are not available in Trino or for improving query performance in situations +where running a query natively may be faster. + +Find details about the SQL support of DuckDB that you can use in the query in +the [DuckDB SQL Command +Reference](https://duckdb.org/docs/sql/query_syntax/select) and +other statements and functions. + +```{include} query-passthrough-warning.fragment +``` + +As a simple example, query the `example` catalog and select an entire table: + +``` +SELECT + * +FROM + TABLE( + example.system.query( + query => 'SELECT + * + FROM + tpch.nation' + ) + ); +``` + +```{include} query-table-function-ordering.fragment +``` diff --git a/docs/src/main/sphinx/static/img/duckdb.png b/docs/src/main/sphinx/static/img/duckdb.png new file mode 100644 index 0000000000000000000000000000000000000000..88ea6a209750b30622b679b7c926690ade817df3 GIT binary patch literal 12403 zcmeHtWmH_zvStH~yKA6v3r?`$?(Xgcx5k3IySoPn1Shz=6C8rOyIY3bd*{77v)-Fu z^YfivyQ{jYzFoCXowd%|yT5Qnc?o0$0t5g6fGi~`s{Ag8{th^p_wOH?l{o+aM4Y9F zh@zB;2uRV%-rUm03;>{sFOD0f7j?uM6kz+poCb}E7^!1QO7YSBuqJNUm>_9kZ{ic` zr`X@wLJ7&ir%6j7!q^vBO{pbiX(LKA6EmSD$%V&NnZ&_~vYAJ)&%W2^P9}H67RzrV zt)T2-o%nnRTVp5bYd&SM4-f=!N8%GFsG$;Z58v0fZwX?*YkZKV|Lrwjhryb&FiP(= zj`|k7e!bo!KsbuOQHF&=@GTkl3=@TEPp%(&6GkQmV@)N?;Zf6ZcGua${39!k7gL4U zhdVlNhRxbJ=Ne(hRpX|NX3VoC@N26k;ZJk&pf=FSe!dNGXXs^JM z<6#!Q=eInbfu<~4Jqb%dsiomQ5oi=7S==rcB0R{oPju1(|jmrH@|&YK6;i zS3}7!OD`*Dt(QW&=!kOCT8cmaV3o4xMVW+E{0x7as>t`cey`hB^83|W4vvo4=D8Tz zOXs4){I~t*pRXPCqm2&Q;MccV<9nEa6r&wT4vu`adMGv$9KWv>akUXA?=7-1)08ro zlLOGb>u>-d1O@==U4wZ403Zke(Erc@0DcI<|Io!lEdjuP+2!Bm-%Ij+yd(Z4A@d;r zBVs5I>Ob|rp&vA^PTvK*gQS)-0Dy@3cR&C#vhV-^NOMb7O|Yh%ERTu3Eu)dCy|Edi zyRE}t6oA*A=Uufm0~>+dZEftFdEEKP{t?0RuKzVNk%9gZ0=DKO)09&LiP$@tfjAhM z8JWrW5kMdiual`ckFu!vzu@n`_{hG1!45o3Om1#&jBad<_D&W|EZp4OOw6oItgH<0 z5e&{Ac3>lS20Lf+e@^nh=Mgn?HgU3a09)GIf&R{GWNhyO<|8Bfo9I8Tf7S`MH2=>` zcFzA=*82jP{(6{L7@3*=E0~$P<$nYF>-i__AM5&OI^MsN@n|ZWIosQ~{4I;0la==$ z8U7FBf7bU;fP#~y+4~Cr0bu!8{J-1&?|7~MD#5=S{|)fBP98-|cQYF;QA=AhJLkVu zu`@ICGX0mH{}QSB-y;9b^DiVX)89_~TW9{+_J3I4`_7NR%k&@55I@4Qh07NJfIe4B zR7lkw;?x^4Nl(LL&G*cD4K=z5Kia(Lmt+cHkco(f^kWZMwXuV6b+nYGVg}BFyIHbW zowPRW)=%~{_DaiPI=mya5qcP+YV1&U$g@2s;@=p?+xK(oCfA<5p8wpf{ptME9!pII@%9l9GNg+Y&NPMFq67&- zBrn7dy-&0o?TeoudJzPd>o6K~KpGMrB=Wmt2MjXeLyDs%zK20TipRlhEK-Ve!5Om} z!m$f>!A%2*T20~bgk)im{9)cDfuBh#5(IhJupj9Z0D@c+%Hbq|)DXym&kQ6083HW< zF?y`PC=eA?h#=IvMDD7Q`J3Fd>wl8}KW=+7%mQPLjQ;91u)!o}o5XjUiFY=z`t-tj z;nL+R7v6VJZ>U z4J*a2X4Z*IFXSh`ouT8{sKx_%7~(T=;6V2)zNpbxhzZ0kPubkz@{>SSNrm&N^_+(- zINS|MOe1{vDypH$#Gq-No}vp|QhuPOh>>@nMuem@%5MO5^(PN7_x$LWFkt|zM~C~8 z;MIma|F#tAxs*(#q>eR8-%tc9J~cjXA7W8}J*uxPD|x>$wr@w$NK=4&aY2}Y3lO`{ zY*`y-Pa+_W25Cqm5#*l=X^Dwrb6E6&9o=6h7bJgFkH3)*OB%wpO%gmG$dv*rk%znH zR?ybDp8gG`G^uL3Rm~KMc99ILz+p2a#c@1bQftDhd6U|H3_2E8+j z^!p-RVnvIuq~xUV5U!fB=eyO{z40MMk}0XIpVMSS%JBp?`4%3i!$a(Je&ytJ zZUjp_UXNF6PgD0MI-SP5k(3|S#dURb!NN)}BL$|l-0&{}?&)nw(g#}K^=dsA&>xqs z6KkzCv|M=&;@d;bd-{O9@te_F>Djlt4~{p~Ct8apme_f`_&ejvf|D3;*9wP9DG9ZK zRHP14n0>dHXu5FelllQxfKU;xpYyAWl>7KL(p0<9%M|Z79jwuZg-wh&aEwJZFsLPQ3g%G) zKKg-d@>#wN@0z{}mY;kWT%^qtjcB{pE8mr~*JoI0_fq1h#I>AE&i56J8~yTHj%9@# zQ0=_w*1mxuMrjaD9bx#Ygb7;j_=#FG)xNZ_Wx*e-9TyjQIRlH29BMvR96^DPo zD1;pjO#vT%0?txlD_=D?kQ3TSpbEoejoBXkvH^v!otn5k$eW%H3jy-EU-2$(dyB5C zHOr*7J;h@kX6J#eU!f3$ZizrGn-rpk!?s*06dG0vL^;esI&0CcTPzpMEb-}GS3PTo z96GukS@CK;AsHP`T+d95^?vc=t&8N-jEu6O8S3t3Qzo-7>v|^gsTnMrgc4w2NA}ld z;AH58W97L;e;EC=Y86B&Vd`Q4^HoF~g&52?uz%JnzV;AOfrOmfwc88ojf%))n-Pwa zf_w8PsVC=tG#$Y&nW7I|v5crJ+0zzPU-f^i+oi5?K%FEt5{JJy-6jlSd2unMm?Jk) z$>Dtzja+$-hq@}v#iCp|lJd=^hNUWv1r@+(z}(RyP1LHn$xSXxV8Nl8%{xj%MJSL4 zG4pJWeZMf{J3y#LFmqDF*F+Wbs3N>y_q;c6-nGrKA>L(pF4AT!*FPzH1K_2p!okK) z21N~+tY}ZUZw|L)rpN>e$FjoWRUMMr!05XwqMV+`NsyEc*`DuDmD3d#PDLBCYovB~(PvYS8cQ7$Rj;d|;kCSfA(vac7`jcT{#Ra5L(lmMVantU! zFq?AIHLlIlyZMwFR5n{d>{AuD`K4rF?70~oKz|5K45|U1<{G{tQ0Vw+akh%P#xxA) z`h26g_wphcnIj8?QoN$hlQH8$Dwmvd!%W-S~EVI zri_ogIU*zt?VtT7Yiw&*Yw;ZHH{cb*Cy$;AAZH9w$~q~J)@Aom5fGgA!wkZ6hV~IF zy)tDXZ&oBo>3o)tY}?;s;ieH=s%N2o9#AsZMV$>8FozJ~N()X!f|X-<*iFQ5=t3>I zb(DgYe|XFb*Dky81op6ZRDIO2JC3vGL!ED=CcHXl^3+!k6ylULJ8Ch zC#=4(QGpum?(xI-LOG)HX$UARtUaZ)b-MnnWA=49_55wUz-A>ae#wJG{JABF}P3l?OOo0f_WJg+~MX7EdoPXsL}j^r25?n!{~=UUW#XbNCK~%nYjJ+O^B|5 zo8%o?y?z~=O4T&cT20}7A9YoIvy}Y!@V~OhaY11`&siQG=O!CG?aEF6UZO?yb=H1b zFx~9f($%abPD;%~EVfXv+^#4+D74j74daJ1BL^>Mo59LSMTr)4A5e|}>-{kCOD zn0!;^Vmfg2Qt5fJX^%{(heD<+!L_*!$#nUz3!^sQmqMa57!&Q6834vnulsv$!Sf=16h2hIG z0pjHQwVcrVQw0`6xnBW!WzDIsst6#wUVR`YI?uRLzhb$x^TqINhEEX;3h~$gh zg6ETkf7`)P9;YJ(99io9$ZxybopMy9;kQ7un>am(RMPY#5$W%B9YgQ}@;;~qlzmNE z(jUhf@yAFX^odOzNt~gz@CkL2VqZ70uioyJ5SJk6u|2UlStJ}soy-|cAqLq@CGh0` ze8`D?%``LT(o%0|G2%Vg=NJ(|bs4hnc-C5ChYak~r#ah!CPD5ngwQp4z_bm@DVGVb z_?SfvNo(oE#iBkfjVg$iU@{|_x6^d&);RRx{1gh)^^1rsDf(>!w33&$$B{4}xBJu0 zx{J~_9|eomj-pR5OGvL6Gbzn!YNj3SkaunyBuscjl#Nxlvq>7v6)+uoWg$j@w{>lB zUqK4Hut*B&q);HMFCf^=$(EK*tHCl2%pz|fpqPA#>-Bicc|Yv?_9Yj5`=a?BFQyvR zUj=>#DzXq{P!GpI+inzK`83dLO)UX=0VsmcYw~=Ax^mV)ky<-XK3(Sr&ip+1iiz4H zowphu!-Ae`ExnjJ8{)uK>)-=NkpaSjUwFHO6APBBF9wiyE4Paw&5^uamxG8fw^Gdw zhmB&HG7|XjC05Rr^3o!|!1gf6|CAuR2O0d2xlR1qj(~&MTz0%cIGztD3iX}7gXJ7eX37krb%V4c zPg5wF9^dL#RwK*;`A8F2p;w%6H^oIbb9oWnByo4D1Q=`0BSnpptky}LPjnx1r9z7n z88bp>`2D!o3^0L)4Nn1~WEc*1o7AL<;p=Ls4fd`-XWu$)N26t35!aXA$Zd&|8XI}T zN*I6FF2#luiAqWorDjdViw)gLG~Y1f zs3VO#AnwD?5dj)JJs&z#UUj90n3|f3(}~4FrF=LllF1H22=puhtX2Ad?fRxEEQvxu zpenSFD=DM^z<}=Vt)}-At7_d&Yde3nzSZ?43>RvL68*kff;;iBIqj5s96RaW>!D{D z!6bj%Z!tmti%MLcT?mdG7SX%i!@W*@hk^5E7}?%0kyti+_VWf*2RpCESUrbeVFtDg zvVk1%y1=5}?pHHVaM5u{04ZP9a*zer(1ZKIHo0VHZXtPB;H6b)3H5mG2|~Myd8!9F zS}dNA!2t-nv~BE6{$b2%LMnd;Dj1gjao`k-`R#Cvg#pQzrFTY@NtQ%I*Zo)cg|JGe z0==2aHNUm__dq~Nj=m&R&xg)-lzDQ9IP=e>g`;6{S>+A-LfSplohY2ck#=42BMW+F3}RaxSNW}iK; zw;SRjMQHqL)TpOUZeKRLe2oFEC!VRJy_}w82!T{cL~LJ4b|FpYRlYQab(MIXwP~g* z4`etcg25@sC+LASW?hC&LV+ZfB){6=e)RzOUsjgVKRaxc&YoV;SBuR)AQ809axp{F z2&pS7a#NV4BCm>hFHT|3p!@ssEPsx*cO&^2dGU3l1^bq>u#N`uH2MVk5Z|8y{Q>{& zFnoDXizJWViDK~N7=w_j;n!-k+(JzFOWpcWZ-|`)Wf#8{7}YH>^Z_D@kT)U&ma+np z=1P;nV`f~$>0ZKUgI=RX5B;IA$M>7^^b_nywQ!SI2ng|)ZWpZ+vTzqRezdD%A9P9pNX``Mi6$jqNj&m8@HCm8?aHrZkrZFYV=eD9o2CW8^n4k4UczDD39* zI#09lqY9#QGJXq`CgiOlyFf;|Ymk=y2(}-wk!Gk_U$a-atgP{aS`Ms|B;08WGw%+| zAH@>*xCzmF(5rRxY&}`yIzOHDlX=ZUxD%k5wRg$b##m9T9JW@s1DFpKss^*Q;w(~q zpPu_kWB)kkLn&!tq5iF!EPmu=Ntk04Ufp2$^6N6Sy2pI=A6DGVyV zl&jWk)h*L2@HHsBP2QL6)C@K+ECy5@S*PC3M-g#JVqC|9<9VjLKEfR?&auCo-L9wwL{*InQWo6-O$_nrSSyi96MfUFpe8R z@hP~Toy2v%5-Pl%>P=eazi-KgjArY8$0w9X4q+!KtF*b33{${K&;5!S&)J7;zSK}3 z#(lF9d|J-!eZd=xwP7gBa=4I2dIY9A<~<3@1=xy)WU%QKyCcBiWREeO;7n9UBo9lX2mT6_&#$@BDXD}U%TEA=5Z`?ykx<(j2>C}_O z^!Q2NGO@RVPJZ%s=B~$~KYj34YA15)EA|Y3^5}U9eRI{JI`^cYu^b_bwcAIqPrnf^ zNHVn;?Lz(_J*b?^&PHvUj?f|1lxmC4rZrNU^24}2vN00!xx}T7Rl*nhHeF3BVcf|#J={4+IG<%jm@tDOtMrQy5^P;>ZDIw#C=mi1OOgZ^hi z*7+y~9zK-5ABo;dp)7HG)|t->e1z~1k~CM6pUYOftRs2vG?>g}W$jWf5D+wXHOHs> z*&ZU9C?tE}P$Qu}5v{&HxT{rqTT%vIQcyHCy5Lt~BlH-V zHf7*3Yt0f}tq!^(U43lyoI_2k|h3;u0qn*CAg zV4uTFu{@lRRe#rm-_Hl(wL?4|4mT4I+81}Om${l!;xBH!6C$D5Qst-LQupiWfXwXi zh#WaOGm+WH=r}CSa$?+$5G_Ti=@5e~qCthY$Vla_xI2yKnm@!&Z36QJ$G==Azp#T% zXu_24L7P)eboQr%*WUCeDt;>{zp1`Zn+Qh^gSE5=K8e;3;Ez8Rv(-fJDqU zSht41-{Hvi88wgs75S_%fq2JEk@HM`_;QV?7d%2I?N`eSW5eaX{IR$w3_ZJ)YmkEq zXx?v%(|X5lFHOg^PwL%hBo2RkbxYi=@W2hA5S>li&BxKROu($apGd=r((+VF#o2MZ zbydmxtF4qzQcssJ(+`!vpc=b^C&zdOUc2h4MQrVvsWIO{O{?zPJfKf?Rt5I92CGNQ zFbRYCvLR25*OiPs@YO@>cd+(da}*Y7g^tg5w*U9=qu|lUz{UJc5Y;E{t7>cF1%JaL zge;sXL}xvoC}FKJb7!db>Orc&1T^O6bbnmOkV2TW)UqmPsY@A-8BU~FKdLMh6UX*) z7E-=2WB=IY2mItlGnvo|ZD`M2))IADgaDYV`b#zBNJ>SkPJgm-O*pRx5jVMCy$U>#^H> zL5G{e_}RwCcfl*Tp)OgnQBG9AK0u@xRINhp%fb)^nOx+Xr>7sO;ZV0fG{`j}1rXAd zsw}WV&Yjo9L1@nW72OOulhxkyT~^WQApSW$s|3<|E?8D32+~;vp*|8HsScdDnM)bw46OnDQ)VI%?SvK{j472*>Qh6+Ja#3jL5I$p(9$<5yfZ!2?|hM|5-+N-GM zAvHO#khmOODKjiEEU~Fn^1L^8>`vSYNQ``n?=Rz&_(Kl?XHOdk7@P$$vI13n3VBpD-_czW3)83Bqgb+np6I)naP{HdPQ zU^)T2;JlgRvKaxpCr+)r$A+8*(&G;cRc4pbtVW)ck3$oU#5f6S`gR+EBg(@)@IsZJ z1RPi5^AI?l-8!hEADlE&>>~I5ZifOcSZ;#FA0`^Nv+vT_MKy!XP{G;Z^b~#|FkaaY zm~MRyN_OY=)ehP56f-Lr1f391UQ)EKi>ipjxfui7ym-b&RAbnvFd3SAhr5ccIh*Bb z)!(K*G4#lrR~p6Bqmni@7_6{80VWHWXAE8a`2n_m)fM`5nh=kT3i8)MV-rVQ!m-;A zk^rHH#R?;sg}|GeCSy4gXms8&;LI3$2T9hi8@<4jh{tUw6kr|-kmv{o4!jlAB_~Ic zO7bV}8a^-R3_=xGV{+gXqQ}^t4W)(^WWSN^r=XtN=1{6~_Qo{ho}%B>IjFPu0FQwp zxYSg}5i@7nf&Swe&RMzD^#deVV7?$r-9eR)u zi0Mz1ZIE;uNc3&0ScIHe*$HxFe90HJDt%kuVzpsqu-Cq+Jcy3#+pXn&wzxw>{1*6Br&Ry#jcN;=PkD&YV$A$NT>>%mEoZ0IHr-& z-MT+@bh&?eAWqiS%YFmYjH1Fl*ZoD&`qcD(xGMD)*T!C2sQbLERGh5NM0{P(1705J z_UQ*w$zX`4no!kWWzxh>kYp@wbYf7DCra$?X08V;utDO=X6v{+w&CV-HzpAb8fzKJ z3*yF48p}*?sa*|EX2qedJbu(K)9A|d6hd?n+BA-H7h_*rF$f|nhvI`aZ;}Lrl;jS~ zDwWCHqVWwz=Ybj6;J3aK;9j%7nzq(r<%KyPPv|zDtVtcuWkuPIWSbQ*uotqDMuS zq$V;z8#FO~dt7^po<3G21aQPhUye#K-`+RmWSpIego6-tcSKr#1Dn9Y>)2ZleAtl> z@zZjI0C5;5?jI@3JK@x-Dyh&-HtekW=={ML@Virxo1gO0WQ?O$P4l>%8`gt?dq1SA z?}D#f^j0&prN0+>W6WE6f~;rnri7sXJ1uOH}0V5kt2^ zJ|fu^ajHZe3%t7j6PH@$wcBLlo0RCEgY;w4h#FHdQEY0wJXMB1$NBOeWZ`a?L8qhr zik$u!rGTG`l*^7fo6tv4dY_5hq?g z(ypgj37rp-tn-2`thU0$Dl_jJ=EUGBZh~7dJ$-lZdh#N1P^@{C09Po&H_BK&xVNgE zTUeWbCJ!AD_>k5}BAo+IN*Iuy9O49x zwk>Y0*7?LklPOvPH_-QhI@oUh3lP4iN%xp1N)_b>B6v#0z%S@vzBrG>;xaOAO;fk; zb1_bN?NH->kWulfio0*$diV$9%Vx`3UdJBNhkhp6(PRAu@^*&MDb-p^Cuy&~2MT@< zl(>U}?&er0vmZ8pLVoX%#V&wU%k5~s?u8moLg1n{;IJ@Oix+}7Kw80_aYPl6#8ng* z`q@>+2K}PCxecb@bfq%n+Ngu_!nJY~)T>7fi9WL?%*dsYEcq0_9A#F=-xA#&^}FIi zaF8j^oIg@{(DnykKv$#u(j$i=`Bvh)%(T$9rKPoC5Z5a)6(%U4(wNA`&lZfG)44(! z@;kirC#ta_A+ZD+yRcObp9!Qzb5}GeH$OA;tc!cZiCC0;#dnOnWJFY+z=^q=PWBh0!jYVu~_a)<#_iHC43^C}e^2bb@{(tsK}u4iyy zkPbmno)(87Wk7#39LtBE`{HGmPZ9&|TS9vOQ+OK@cfF9$`2^pSB6dY)nd;%n{!l)( z8oay|b}JMZmyG1L@=aN9Z*VSQ*qaL@V3Zm~zlxF>gUby5zafTWu>I3n)u%WpZ~X{my&&nH!dt zX@bs5D(#Xn%Rs{np|F}0T2i=^>$EB1=fj&wSrj`gvO-OPiy!t1BS&i-9&_{>Cck~% zLwg)23;)2I(g{|Hw7w~CIIb!r7d)-Ypl|aV<{u%1ti`kK9dYKYgVx=-=>Kfp<(5T0 zbhLx~mYal7?$RK16&c|ph2bR?rrw2)qncYAW)R~sVF zW)680m90K#bv=nKerr9xt2O}kFCzf{U zgcrPU;z(@MLTGgTx`{XLGrBt_dj*y&M zXuB=7^guTon5U5APX!EnyIEgH;34A+HbfR{UoFm^e6Bcb~cEY-moroX+?2h_Rly$dM*JVAMy ztqKzJnd4^?IY7g<0lnJB6!;t!4#gQQJUR|0C&cN^(X}4gYA(26dOz!KLY$xcAt$^} zll(ZE{F`T|&VVSvpb5qqYR*ybE`9e+XiDy&U`u}@K%MY2dk%-ZmWRK(Ly4yV*AH7b zJoU0l|6UuT!jm<<s@{L7V zAmU}Ek2yV(deP}87#E)JkKywmwuj~i6cgdT`dDc?Z5w585Z6tm$tvidA9Q9=7&yHSjk;ptHXFy@By0S zCm}dQvEWxYse*#sVug9KpOs@$5QL;8yk4qy?PVHgFKF}9x($BLiOTvk2&%Y4c#W7_ zG0|Oz z@exqp0_b4uLZYhOR7k`-u@h&$`$Iq0A0@t_STNBlpGCq z4FNbyK?~um1tWT=i&&w>5u)HCJ5oRY1Mo07{HNYs?C#F+`S|c4zmH(fQvasPUn+y4 zBv+xQ3T>g49a}0Go53h`EKr+)y3PStn+~P9)*GMvb?xuufMeY^zrf+bUOO&)iAI2XPFAWQmE0tWejse#o@o0g`aXQ;r3%tc>(E zQxZAhcY;Y+A>7~zf2cTJ9~N(0CI9(omsQhkGV{E^7)};OZhYg~A{FNv^T$k14bv~Z z?*xm=o-r@+l{pq@$w(?BWWNNB>-7f{X~Z_r^5{mgXAoI2cwmABVB{*caGEDTgNf9d!s5 zW;H)>)WL(8fvrLVpNrqs1!g)hRZC8UHYeTrDOg;%VxjXVtoinLo7H7{ou8G-6BBed z>10$O7K*}tq`D>IqE^*_PSpk(*K-~+0Z$)g*HgODn0+GNtY88FA1`uzH<|6NYQ2;e@jnqETj+$Ep8pXr3yv}K2;6Mmi>M4gKDH%qh zFs$}|P2lCpGzGr%t(<<2k50gU2*zZjqO(Ekq9hGg zh8z@Ygo2ygj=L)DCFUeeh36a$VLm3XAMjUne*8Q$gYQxQb`n4PwfIUFgG-p_abGbU z0qwl|6O4YTi3Nj8J60tnrxguRjZY96x;1G=Vsl}{C_NTmvTN3s{R=~WHx%`hojc}$ zn;VmN4ueS7a`u!jE;1l-t^?|#GKyG-fyIapMJ5IG8D3tp>ZTyer!!% zLJK2$XEW}g>A+Zucflp>E!Wjb#Q338IE66;vXdJErJPT-mdGW#;4rxRQiL+lnFETN z?-(BHE#-pI|cW&gguBZCl(7Q_%rN%7I{GI-2S;`;-5Ocq~hDd)RO-H@w9B;5Ni~XN+{{P*bhd;R#ANKtL6jOgG O;8J4pqLspi0sjk2441tC literal 0 HcmV?d00001 diff --git a/plugin/trino-duckdb/pom.xml b/plugin/trino-duckdb/pom.xml new file mode 100644 index 00000000000000..7b107d96f041d1 --- /dev/null +++ b/plugin/trino-duckdb/pom.xml @@ -0,0 +1,206 @@ + + + 4.0.0 + + + io.trino + trino-root + 458-SNAPSHOT + ../../pom.xml + + + trino-duckdb + trino-plugin + Trino - DuckDB Connector + + + true + + + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + io.airlift + configuration + + + + io.trino + trino-base-jdbc + + + + io.trino + trino-plugin-toolkit + + + org.duckdb + duckdb_jdbc + 1.1.0 + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + io.airlift + log + runtime + + + + io.airlift + log-manager + runtime + + + + io.airlift + units + runtime + + + + io.airlift + junit-extensions + test + + + + io.airlift + testing + test + + + + io.trino + trino-base-jdbc + test-jar + test + + + + io.trino + trino-main + test + + + + io.trino + trino-main + test-jar + test + + + + io.trino + trino-plugin-toolkit + test-jar + test + + + + io.trino + trino-testing + test + + + + io.trino + trino-testing-containers + test + + + + io.trino + trino-testing-services + test + + + + io.trino + trino-tpch + test + + + + io.trino.tpch + tpch + test + + + + org.assertj + assertj-core + test + + + + org.jetbrains + annotations + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.testcontainers + testcontainers + test + + + diff --git a/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbClient.java b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbClient.java new file mode 100644 index 00000000000000..ba5ee3ebe3e29c --- /dev/null +++ b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbClient.java @@ -0,0 +1,300 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.plugin.base.mapping.IdentifierMapping; +import io.trino.plugin.jdbc.BaseJdbcClient; +import io.trino.plugin.jdbc.BaseJdbcConfig; +import io.trino.plugin.jdbc.ColumnMapping; +import io.trino.plugin.jdbc.ConnectionFactory; +import io.trino.plugin.jdbc.JdbcOutputTableHandle; +import io.trino.plugin.jdbc.JdbcTableHandle; +import io.trino.plugin.jdbc.JdbcTypeHandle; +import io.trino.plugin.jdbc.LongWriteFunction; +import io.trino.plugin.jdbc.QueryBuilder; +import io.trino.plugin.jdbc.RemoteTableName; +import io.trino.plugin.jdbc.WriteMapping; +import io.trino.plugin.jdbc.logging.RemoteQueryModifier; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.type.CharType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Type; +import io.trino.spi.type.VarcharType; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.trino.plugin.jdbc.StandardColumnMappings.bigintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.bigintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.booleanColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.booleanWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.charWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.decimalColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.doubleColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.doubleWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.integerColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.integerWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.realColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.realWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.smallintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.smallintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.varcharColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.varcharWriteFunction; +import static io.trino.plugin.jdbc.TypeHandlingJdbcSessionProperties.getUnsupportedTypeHandling; +import static io.trino.plugin.jdbc.UnsupportedTypeHandling.CONVERT_TO_VARCHAR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static java.lang.String.format; +import static java.time.temporal.ChronoField.EPOCH_DAY; + +public final class DuckDbClient + extends BaseJdbcClient +{ + private static final Pattern DECIMAL_PATTERN = Pattern.compile("DECIMAL\\((?[0-9]+),(?[0-9]+)\\)"); + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd"); + + @Inject + public DuckDbClient( + BaseJdbcConfig config, + ConnectionFactory connectionFactory, + QueryBuilder queryBuilder, + IdentifierMapping identifierMapping, + RemoteQueryModifier queryModifier) + { + super("\"", connectionFactory, queryBuilder, config.getJdbcTypesMappedToVarchar(), identifierMapping, queryModifier, false); + } + + @Override + public Connection getConnection(ConnectorSession session) + throws SQLException + { + // The method calls Connection.setReadOnly method, but DuckDB does not support changing read-only status on connection level + return connectionFactory.openConnection(session); + } + + @Override + public void renameSchema(ConnectorSession session, String schemaName, String newSchemaName) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming schemas"); + } + + @Override + protected String escapeObjectNameForMetadataQuery(String name, String escape) + { + // org.duckdb.DuckDBDatabaseMetaData.getSearchStringEscape returns null + checkArgument(escape == null, "escape must be null"); + return name; + } + + @Override + protected Optional> getTableTypes() + { + return Optional.of(ImmutableList.of("BASE TABLE", "LOCAL TEMPORARY", "VIEW")); + } + + @Override + public ResultSet getTables(Connection connection, Optional schemaName, Optional tableName) + throws SQLException + { + DatabaseMetaData metadata = connection.getMetaData(); + return metadata.getTables( + null, + schemaName.orElse(null), + escapeObjectNameForMetadataQuery(tableName, metadata.getSearchStringEscape()).orElse(null), + getTableTypes().map(types -> types.toArray(String[]::new)).orElse(null)); + } + + @Override + public void renameTable(ConnectorSession session, JdbcTableHandle handle, SchemaTableName newTableName) + { + RemoteTableName remoteTableName = handle.asPlainTable().getRemoteTableName(); + if (!remoteTableName.getSchemaName().orElseThrow().equals(newTableName.getSchemaName())) { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming tables across schemas"); + } + renameTable(session, null, remoteTableName.getSchemaName().orElseThrow(), remoteTableName.getTableName(), newTableName); + } + + @Override + protected void renameTable(ConnectorSession session, Connection connection, String catalogName, String remoteSchemaName, String remoteTableName, String newRemoteSchemaName, String newRemoteTableName) + throws SQLException + { + execute(session, connection, format( + "ALTER TABLE %s RENAME TO %s", + quoted(catalogName, remoteSchemaName, remoteTableName), + quoted(catalogName, null, newRemoteTableName))); + } + + @Override + public void commitCreateTable(ConnectorSession session, JdbcOutputTableHandle handle, Set pageSinkIds) + { + if (handle.getPageSinkIdColumnName().isPresent()) { + finishInsertTable(session, handle, pageSinkIds); + } + else { + renameTable( + session, + null, + handle.getRemoteTableName().getSchemaName().orElse(null), + handle.getTemporaryTableName().orElseThrow(() -> new IllegalStateException("Temporary table name missing")), + handle.getRemoteTableName().getSchemaTableName()); + } + } + + @Override + public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column) + { + if (!column.isNullable()) { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support adding not null columns"); + } + super.addColumn(session, handle, column); + } + + @Override + public Optional toColumnMapping(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle) + { + Optional mapping = getForcedMappingToVarchar(typeHandle); + if (mapping.isPresent()) { + return mapping; + } + switch (typeHandle.jdbcType()) { + case Types.BOOLEAN: + return Optional.of(booleanColumnMapping()); + case Types.TINYINT: + return Optional.of(tinyintColumnMapping()); + case Types.SMALLINT: + return Optional.of(smallintColumnMapping()); + case Types.INTEGER: + return Optional.of(integerColumnMapping()); + case Types.BIGINT: + return Optional.of(bigintColumnMapping()); + case Types.FLOAT: + return Optional.of(realColumnMapping()); + case Types.DOUBLE: + return Optional.of(doubleColumnMapping()); + case Types.DECIMAL: + String decimalTypeName = typeHandle.jdbcTypeName().orElseThrow(); + Matcher matcher = DECIMAL_PATTERN.matcher(decimalTypeName); + checkArgument(matcher.matches(), "Decimal type name does not match pattern: %s", decimalTypeName); + int precision = Integer.parseInt(matcher.group("precision")); + int scale = Integer.parseInt(matcher.group("scale")); + return Optional.of(decimalColumnMapping(createDecimalType(precision, scale))); + case Types.VARCHAR: + // CHAR is an alias of VARCHAR in DuckDB https://duckdb.org/docs/sql/data_types/text + return Optional.of(varcharColumnMapping(VarcharType.VARCHAR, true)); + case Types.DATE: + return Optional.of(ColumnMapping.longMapping( + DATE, + (resultSet, columnIndex) -> DATE_FORMATTER.parse(resultSet.getString(columnIndex)).getLong(EPOCH_DAY), + dateWriteFunction())); + } + + if (getUnsupportedTypeHandling(session) == CONVERT_TO_VARCHAR) { + return mapToUnboundedVarchar(typeHandle); + } + return Optional.empty(); + } + + @Override + public WriteMapping toWriteMapping(ConnectorSession session, Type type) + { + if (type == BOOLEAN) { + return WriteMapping.booleanMapping("boolean", booleanWriteFunction()); + } + if (type == TINYINT) { + return WriteMapping.longMapping("tinyint", tinyintWriteFunction()); + } + if (type == SMALLINT) { + return WriteMapping.longMapping("smallint", smallintWriteFunction()); + } + if (type == INTEGER) { + return WriteMapping.longMapping("integer", integerWriteFunction()); + } + if (type == BIGINT) { + return WriteMapping.longMapping("bigint", bigintWriteFunction()); + } + if (type == REAL) { + return WriteMapping.longMapping("float", realWriteFunction()); + } + if (type == DOUBLE) { + return WriteMapping.doubleMapping("double precision", doubleWriteFunction()); + } + if (type instanceof DecimalType decimalType) { + String dataType = "decimal(%d, %d)".formatted(decimalType.getPrecision(), decimalType.getScale()); + if (decimalType.isShort()) { + return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType)); + } + return WriteMapping.objectMapping(dataType, longDecimalWriteFunction(decimalType)); + } + if (type instanceof CharType) { + // CHAR is an alias of VARCHAR in DuckDB https://duckdb.org/docs/sql/data_types/text + return WriteMapping.sliceMapping("varchar", charWriteFunction()); + } + if (type instanceof VarcharType) { + // CHAR is an alias of VARCHAR in DuckDB https://duckdb.org/docs/sql/data_types/text + return WriteMapping.sliceMapping("varchar", varcharWriteFunction()); + } + if (type == DATE) { + return WriteMapping.longMapping("date", dateWriteFunction()); + } + throw new TrinoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName()); + } + + private static LongWriteFunction dateWriteFunction() + { + return new LongWriteFunction() + { + @Override + public String getBindExpression() + { + return "CAST(? AS DATE)"; + } + + @Override + public void set(PreparedStatement statement, int index, long day) + throws SQLException + { + statement.setString(index, DATE_FORMATTER.format(LocalDate.ofEpochDay(day))); + } + }; + } +} diff --git a/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbPlugin.java b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbPlugin.java new file mode 100644 index 00000000000000..fa52a1365933ce --- /dev/null +++ b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDbPlugin.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import io.trino.plugin.jdbc.JdbcPlugin; + +public final class DuckDbPlugin + extends JdbcPlugin +{ + public DuckDbPlugin() + { + super("duckdb", new DuckDblClientModule()); + } +} diff --git a/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDblClientModule.java b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDblClientModule.java new file mode 100644 index 00000000000000..f243135c9f4e3f --- /dev/null +++ b/plugin/trino-duckdb/src/main/java/io/trino/plugin/duckdb/DuckDblClientModule.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.jdbc.BaseJdbcConfig; +import io.trino.plugin.jdbc.ConnectionFactory; +import io.trino.plugin.jdbc.DriverConnectionFactory; +import io.trino.plugin.jdbc.ForBaseJdbc; +import io.trino.plugin.jdbc.JdbcClient; +import io.trino.plugin.jdbc.JdbcStatisticsConfig; +import io.trino.plugin.jdbc.credential.CredentialProvider; +import io.trino.plugin.jdbc.ptf.Query; +import io.trino.spi.function.table.ConnectorTableFunction; +import org.duckdb.DuckDBDriver; + +import java.util.Properties; + +import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static io.airlift.configuration.ConfigBinder.configBinder; + +public final class DuckDblClientModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + binder.bind(JdbcClient.class).annotatedWith(ForBaseJdbc.class).to(DuckDbClient.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(JdbcStatisticsConfig.class); + newSetBinder(binder, ConnectorTableFunction.class).addBinding().toProvider(Query.class).in(Scopes.SINGLETON); + } + + @Provides + @Singleton + @ForBaseJdbc + public static ConnectionFactory connectionFactory(BaseJdbcConfig config, CredentialProvider credentialProvider, OpenTelemetry openTelemetry) + { + Properties connectionProperties = new Properties(); + return DriverConnectionFactory.builder( + new DuckDBDriver(), + config.getConnectionUrl(), + credentialProvider) + .setConnectionProperties(connectionProperties) + .setOpenTelemetry(openTelemetry) + .build(); + } +} diff --git a/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/DuckDbQueryRunner.java b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/DuckDbQueryRunner.java new file mode 100644 index 00000000000000..25a72ae284b5c2 --- /dev/null +++ b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/DuckDbQueryRunner.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import com.google.common.collect.ImmutableList; +import io.airlift.log.Logger; +import io.airlift.log.Logging; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.testing.DistributedQueryRunner; +import io.trino.tpch.TpchTable; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.trino.plugin.duckdb.TestingDuckDb.TPCH_SCHEMA; +import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.trino.testing.QueryAssertions.copyTpchTables; +import static io.trino.testing.TestingSession.testSessionBuilder; + +public final class DuckDbQueryRunner +{ + private DuckDbQueryRunner() {} + + public static Builder builder(TestingDuckDb server) + { + return new Builder() + .addConnectorProperty("connection-url", server.getJdbcUrl()); + } + + public static final class Builder + extends DistributedQueryRunner.Builder + { + private final Map connectorProperties = new HashMap<>(); + private List> initialTables = ImmutableList.of(); + + private Builder() + { + super(testSessionBuilder() + .setCatalog("duckdb") + .setSchema(TPCH_SCHEMA) + .build()); + } + + public Builder addConnectorProperty(String key, String value) + { + this.connectorProperties.put(key, value); + return this; + } + + public Builder setInitialTables(Iterable> initialTables) + { + this.initialTables = ImmutableList.copyOf(initialTables); + return this; + } + + @Override + public DistributedQueryRunner build() + throws Exception + { + DistributedQueryRunner queryRunner = super.build(); + try { + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + queryRunner.installPlugin(new DuckDbPlugin()); + queryRunner.createCatalog("duckdb", "duckdb", connectorProperties); + + queryRunner.execute("CREATE SCHEMA duckdb.tpch"); + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, initialTables); + + return queryRunner; + } + catch (Throwable e) { + closeAllSuppress(e, queryRunner); + throw e; + } + } + } + + public static void main(String[] args) + throws Exception + { + Logging.initialize(); + + //noinspection resource + DistributedQueryRunner queryRunner = DuckDbQueryRunner.builder(new TestingDuckDb()) + .addCoordinatorProperty("http-server.http.port", "8080") + .setInitialTables(TpchTable.getTables()) + .build(); + + Logger log = Logger.get(DuckDbQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } +} diff --git a/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbConnectorTest.java b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbConnectorTest.java new file mode 100644 index 00000000000000..e27b37c283283d --- /dev/null +++ b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbConnectorTest.java @@ -0,0 +1,294 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import io.trino.plugin.jdbc.BaseJdbcConnectorTest; +import io.trino.testing.MaterializedResult; +import io.trino.testing.QueryRunner; +import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.sql.SqlExecutor; +import io.trino.testing.sql.TestTable; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Isolated; + +import java.util.List; +import java.util.Optional; + +import static io.trino.plugin.duckdb.TestingDuckDb.TPCH_SCHEMA; +import static io.trino.spi.connector.ConnectorMetadata.MODIFYING_ROWS_MESSAGE; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.testing.MaterializedResult.resultBuilder; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_NATIVE_QUERY; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Isolated +final class TestDuckDbConnectorTest + extends BaseJdbcConnectorTest +{ + private TestingDuckDb duckDb; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + duckDb = closeAfterClass(new TestingDuckDb()); + return DuckDbQueryRunner.builder(duckDb) + .setInitialTables(REQUIRED_TPCH_TABLES) + .build(); + } + + @Override + protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) + { + return switch (connectorBehavior) { + case SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT, + SUPPORTS_ADD_COLUMN_WITH_COMMENT, + SUPPORTS_AGGREGATION_PUSHDOWN, + SUPPORTS_ARRAY, + SUPPORTS_COMMENT_ON_COLUMN, + SUPPORTS_COMMENT_ON_TABLE, + SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT, + SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, + SUPPORTS_JOIN_PUSHDOWN, + SUPPORTS_LIMIT_PUSHDOWN, + SUPPORTS_RENAME_SCHEMA, + SUPPORTS_RENAME_TABLE_ACROSS_SCHEMAS, + SUPPORTS_TOPN_PUSHDOWN, + SUPPORTS_ROW_TYPE -> false; + + default -> super.hasBehavior(connectorBehavior); + }; + } + + @Override + protected Optional filterDataMappingSmokeTestData(DataMappingTestSetup setup) + { + String type = setup.getTrinoTypeName(); + + if (type.equals("time") || + type.startsWith("time(") || + type.startsWith("timestamp") || + type.equals("varbinary") || + type.startsWith("array") || + type.startsWith("row")) { + return Optional.of(setup.asUnsupported()); + } + + if (setup.getTrinoTypeName().equals("char(3)") && setup.getSampleValueLiteral().equals("'ab'")) { + return Optional.of(new DataMappingTestSetup("char(3)", "'abc'", "'zzz'")); + } + return Optional.of(setup); + } + + @Override + protected TestTable createTableWithDefaultColumns() + { + return new TestTable( + onRemoteDatabase(), + TPCH_SCHEMA + ".test_default_cols", + "(col_required decimal(20,0) NOT NULL," + + "col_nullable decimal(20,0)," + + "col_default decimal(20,0) DEFAULT 43," + + "col_nonnull_default decimal(20,0) DEFAULT 42 NOT NULL ," + + "col_required2 decimal(20,0) NOT NULL)"); + } + + @Override + protected TestTable createTableWithUnsupportedColumn() + { + return new TestTable( + onRemoteDatabase(), + TPCH_SCHEMA + ".test_unsupported_col", + "(one bigint, two union(num integer, str varchar), three varchar)"); + } + + @Override // Override because DuckDB ignores column size of varchar type + protected MaterializedResult getDescribeOrdersResult() + { + return resultBuilder(getSession(), VARCHAR, VARCHAR, VARCHAR, VARCHAR) + .row("orderkey", "bigint", "", "") + .row("custkey", "bigint", "", "") + .row("orderstatus", "varchar", "", "") + .row("totalprice", "double", "", "") + .row("orderdate", "date", "", "") + .row("orderpriority", "varchar", "", "") + .row("clerk", "varchar", "", "") + .row("shippriority", "integer", "", "") + .row("comment", "varchar", "", "") + .build(); + } + + @Test + @Override // Override because DuckDB ignores column size of varchar type + public void testShowCreateTable() + { + assertThat((String) computeScalar("SHOW CREATE TABLE orders")) + .isEqualTo(""" + CREATE TABLE duckdb.tpch.orders ( + orderkey bigint, + custkey bigint, + orderstatus varchar, + totalprice double, + orderdate date, + orderpriority varchar, + clerk varchar, + shippriority integer, + comment varchar + )"""); + } + + @Test + @Override // Override because the connector doesn't support row level delete + public void testDeleteWithLike() + { + assertThatThrownBy(super::testDeleteWithLike) + .hasStackTraceContaining("TrinoException: " + MODIFYING_ROWS_MESSAGE); + } + + @Override + @Language("RegExp") + protected String errorMessageForInsertIntoNotNullColumn(String columnName) + { + return "Constraint Error: NOT NULL constraint failed: .*." + columnName; + } + + @Override + @Language("RegExp") + protected String errorMessageForCreateTableAsSelectNegativeDate(String date) + { + return "negative date %s as select blubb".formatted(date); + } + + @Override + @Language("RegExp") + protected String errorMessageForInsertNegativeDate(String date) + { + return "negative date %s insert blubb".formatted(date); + } + + @Override + protected void verifyAddNotNullColumnToNonEmptyTableFailurePermissible(Throwable e) + { + assertThat(e).hasMessageContaining("Adding columns with constraints not yet supported"); + } + + @Override + protected void verifyConcurrentAddColumnFailurePermissible(Exception e) + { + assertThat(e).hasMessageContaining("Catalog write-write conflict"); + } + + @Override + protected void verifySetColumnTypeFailurePermissible(Throwable e) + { + assertThat(e).hasMessageContaining("Conversion Error"); + } + + @Override + protected Optional filterSetColumnTypesDataProvider(SetColumnTypeSetup setup) + { + return switch ("%s -> %s".formatted(setup.sourceColumnType(), setup.newColumnType())) { + case "varchar(100) -> varchar(50)" -> Optional.of(setup.withNewColumnType("varchar")); + case "char(25) -> char(20)", + "char(20) -> varchar", + "varchar -> char(20)" -> Optional.of(setup.withNewColumnType("varchar").withNewValueLiteral("rtrim(%s)".formatted(setup.newValueLiteral()))); + default -> Optional.of(setup); + }; + } + + @Override + protected void verifySchemaNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessageContaining("maximum length of identifier exceeded"); + } + + @Override + protected void verifyTableNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessageStartingWith("maximum length of identifier exceeded"); + } + + @Override + protected void verifyColumnNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessageContaining("maximum length of identifier exceeded"); + } + + @Test + @Override // Override because the expected error message is different + public void testNativeQueryCreateStatement() + { + assertThat(getQueryRunner().tableExists(getSession(), "numbers")).isFalse(); + assertThat(query("SELECT * FROM TABLE(system.query(query => 'CREATE TABLE " + TPCH_SCHEMA + ".numbers(n INTEGER)'))")) + .failure().hasMessageContaining("java.sql.SQLException: Parser Error: syntax error at or near \"CREATE\""); + assertThat(getQueryRunner().tableExists(getSession(), "numbers")).isFalse(); + } + + @Test + @Override // Override because the expected error message is different + public void testNativeQueryInsertStatementTableExists() + { + skipTestUnless(hasBehavior(SUPPORTS_NATIVE_QUERY)); + try (TestTable testTable = simpleTable()) { + assertThat(query("SELECT * FROM TABLE(system.query(query => 'INSERT INTO %s VALUES (3)'))".formatted(testTable.getName()))) + .failure().hasMessageContaining("java.sql.SQLException: Parser Error: syntax error at or near \"INTO\""); + assertQuery("SELECT * FROM " + testTable.getName(), "VALUES 1, 2"); + } + } + + @Test + @Override + public void testCharTrailingSpace() + { + String schema = getSession().getSchema().orElseThrow(); + try (TestTable table = new TestTable(onRemoteDatabase(), schema + ".char_trailing_space", "(x char(10))", List.of("'test'"))) { + String tableName = table.getName(); + assertQuery("SELECT * FROM " + tableName + " WHERE x = char 'test'", "VALUES 'test'"); + assertQuery("SELECT * FROM " + tableName + " WHERE x = char 'test '", "VALUES 'test'"); + assertQuery("SELECT * FROM " + tableName + " WHERE x = char 'test '", "VALUES 'test'"); + assertQueryReturnsEmptyResult("SELECT * FROM " + tableName + " WHERE x = char ' test'"); + } + } + + @Test + @Override // Override because char type is an alias of varchar in DuckDB + public void testCharVarcharComparison() + { + // with char->varchar coercion on table creation, this is essentially varchar/varchar comparison + try (TestTable table = new TestTable( + getQueryRunner()::execute, + "test_char_varchar", + "(k, v) AS VALUES" + + " (-1, CAST(NULL AS CHAR(3))), " + + " (3, CAST(' ' AS CHAR(3)))," + + " (6, CAST('x ' AS CHAR(3)))")) { + // varchar of length shorter than column's length + assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(2))")).returnsEmptyResult(); + // varchar of length longer than column's length + assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(4))")).returnsEmptyResult(); + // value that's not all-spaces + assertThat(query("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('x ' AS varchar(2))")).returnsEmptyResult(); + // exact match + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('' AS varchar(3))", "VALUES (3, '')"); + } + } + + @Override + protected SqlExecutor onRemoteDatabase() + { + return duckDb::execute; + } +} diff --git a/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbPlugin.java b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbPlugin.java new file mode 100644 index 00000000000000..fac46bf0043856 --- /dev/null +++ b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbPlugin.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import com.google.common.collect.ImmutableMap; +import io.trino.spi.Plugin; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.testing.TestingConnectorContext; +import org.junit.jupiter.api.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; + +final class TestDuckDbPlugin +{ + @Test + void testCreateConnector() + { + Plugin plugin = new DuckDbPlugin(); + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + factory.create( + "test", + ImmutableMap.of("connection-url", "jdbc:duckdb:"), + new TestingConnectorContext()) + .shutdown(); + } +} diff --git a/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbTypeMapping.java b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbTypeMapping.java new file mode 100644 index 00000000000000..29fe88a0169ef2 --- /dev/null +++ b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestDuckDbTypeMapping.java @@ -0,0 +1,331 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import io.trino.Session; +import io.trino.spi.type.TimeZoneKey; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import io.trino.testing.TestingSession; +import io.trino.testing.datatype.CreateAndInsertDataSetup; +import io.trino.testing.datatype.CreateAsSelectDataSetup; +import io.trino.testing.datatype.DataSetup; +import io.trino.testing.datatype.SqlDataTypeTest; +import io.trino.testing.sql.TrinoSqlExecutor; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static io.trino.plugin.duckdb.TestingDuckDb.TPCH_SCHEMA; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.time.ZoneOffset.UTC; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; + +@TestInstance(PER_CLASS) +@Execution(SAME_THREAD) +final class TestDuckDbTypeMapping + extends AbstractTestQueryFramework +{ + private TestingDuckDb duckDb; + + private static final ZoneId jvmZone = ZoneId.systemDefault(); + private static final LocalDateTime timeGapInJvmZone1 = LocalDateTime.of(1970, 1, 1, 0, 13, 42); + private static final LocalDateTime timeGapInJvmZone2 = LocalDateTime.of(2018, 4, 1, 2, 13, 55, 123_000_000); + private static final LocalDateTime timeDoubledInJvmZone = LocalDateTime.of(2018, 10, 28, 1, 33, 17, 456_000_000); + + // no DST in 1970, but has DST in later years (e.g. 2018) + private static final ZoneId vilnius = ZoneId.of("Europe/Vilnius"); + private static final LocalDateTime timeGapInVilnius = LocalDateTime.of(2018, 3, 25, 3, 17, 17); + private static final LocalDateTime timeDoubledInVilnius = LocalDateTime.of(2018, 10, 28, 3, 33, 33, 333_000_000); + + // minutes offset change since 1970-01-01, no DST + private static final ZoneId kathmandu = ZoneId.of("Asia/Kathmandu"); + private static final LocalDateTime timeGapInKathmandu = LocalDateTime.of(1986, 1, 1, 0, 13, 7); + + public TestDuckDbTypeMapping() + { + checkState(jvmZone.getId().equals("America/Bahia_Banderas"), "This test assumes certain JVM time zone"); + checkIsGap(jvmZone, timeGapInJvmZone1); + checkIsGap(jvmZone, timeGapInJvmZone2); + checkIsDoubled(jvmZone, timeDoubledInJvmZone); + + LocalDate dateOfLocalTimeChangeForwardAtMidnightInSomeZone = LocalDate.of(1983, 4, 1); + checkIsGap(vilnius, dateOfLocalTimeChangeForwardAtMidnightInSomeZone.atStartOfDay()); + LocalDate dateOfLocalTimeChangeBackwardAtMidnightInSomeZone = LocalDate.of(1983, 10, 1); + checkIsDoubled(vilnius, dateOfLocalTimeChangeBackwardAtMidnightInSomeZone.atStartOfDay().minusMinutes(1)); + checkIsGap(vilnius, timeGapInVilnius); + checkIsDoubled(vilnius, timeDoubledInVilnius); + + checkIsGap(kathmandu, timeGapInKathmandu); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + duckDb = closeAfterClass(new TestingDuckDb()); + return DuckDbQueryRunner.builder(duckDb).build(); + } + + @Test + void testBoolean() + { + SqlDataTypeTest.create() + .addRoundTrip("boolean", "true", BOOLEAN) + .addRoundTrip("boolean", "false", BOOLEAN) + .addRoundTrip("boolean", "NULL", BOOLEAN, "CAST(NULL AS BOOLEAN)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_boolean")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_boolean")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_boolean")); + } + + @Test + void testTinyInt() + { + SqlDataTypeTest.create() + .addRoundTrip("tinyint", "tinyint '-128'", TINYINT) + .addRoundTrip("tinyint", "tinyint '127'", TINYINT) + .addRoundTrip("tinyint", "NULL", TINYINT, "CAST(NULL AS TINYINT)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_tinyint")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_tinyint")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_tinyint")); + } + + @Test + void testSmallInt() + { + SqlDataTypeTest.create() + .addRoundTrip("smallint", "smallint '-32768'", SMALLINT) + .addRoundTrip("smallint", "smallint '32767'", SMALLINT) + .addRoundTrip("smallint", "NULL", SMALLINT, "CAST(NULL AS SMALLINT)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_smallint")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_smallint")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_smallint")); + } + + @Test + void testInteger() + { + SqlDataTypeTest.create() + .addRoundTrip("integer", "-2147483648", INTEGER) + .addRoundTrip("integer", "2147483647", INTEGER) + .addRoundTrip("integer", "NULL", INTEGER, "CAST(NULL AS INTEGER)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_integer")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_integer")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_integer")); + } + + @Test + void testBigInt() + { + SqlDataTypeTest.create() + .addRoundTrip("bigint", "-9223372036854775808", BIGINT) + .addRoundTrip("bigint", "9223372036854775807", BIGINT) + .addRoundTrip("bigint", "NULL", BIGINT, "CAST(NULL AS BIGINT)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_bigint")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_bigint")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_bigint")); + } + + @Test + void testReal() + { + SqlDataTypeTest.create() + .addRoundTrip("real", "123.456E10", REAL, "REAL '123.456E10'") + .addRoundTrip("real", "NULL", REAL, "CAST(NULL AS real)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_real")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_real")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_real")); + } + + @Test + void testDouble() + { + SqlDataTypeTest.create() + .addRoundTrip("double", "1.0E100", DOUBLE, "1.0E100") + .addRoundTrip("double", "123.456E10", DOUBLE, "123.456E10") + .addRoundTrip("double", "123.456E10", DOUBLE, "123.456E10") + .addRoundTrip("double", "NULL", DOUBLE, "CAST(NULL AS double)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_double")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_double")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_double")); + } + + @Test + void testDecimal() + { + SqlDataTypeTest.create() + .addRoundTrip("decimal(3, 0)", "CAST(NULL AS decimal(3, 0))", createDecimalType(3, 0), "CAST(NULL AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "CAST('193' AS decimal(3, 0))", createDecimalType(3, 0), "CAST('193' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "CAST('19' AS decimal(3, 0))", createDecimalType(3, 0), "CAST('19' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "CAST('-193' AS decimal(3, 0))", createDecimalType(3, 0), "CAST('-193' AS decimal(3, 0))") + .addRoundTrip("decimal(4, 0)", "CAST('19' AS decimal(4, 0))", createDecimalType(4, 0), "CAST('19' AS decimal(4, 0))") // JDBC Type SMALLINT + .addRoundTrip("decimal(5, 0)", "CAST('19' AS decimal(5, 0))", createDecimalType(5, 0), "CAST('19' AS decimal(5, 0))") // JDBC Type INTEGER + .addRoundTrip("decimal(10, 0)", "CAST('19' AS decimal(10, 0))", createDecimalType(10, 0), "CAST('19' AS decimal(10, 0))") // JDBC Type BIGINT + .addRoundTrip("decimal(3, 1)", "CAST('10.0' AS decimal(3, 1))", createDecimalType(3, 1), "CAST('10.0' AS decimal(3, 1))") + .addRoundTrip("decimal(3, 1)", "CAST('10.1' AS decimal(3, 1))", createDecimalType(3, 1), "CAST('10.1' AS decimal(3, 1))") + .addRoundTrip("decimal(3, 1)", "CAST('-10.1' AS decimal(3, 1))", createDecimalType(3, 1), "CAST('-10.1' AS decimal(3, 1))") + .addRoundTrip("decimal(4, 2)", "CAST('2' AS decimal(4, 2))", createDecimalType(4, 2), "CAST('2' AS decimal(4, 2))") + .addRoundTrip("decimal(4, 2)", "CAST('2.3' AS decimal(4, 2))", createDecimalType(4, 2), "CAST('2.3' AS decimal(4, 2))") + .addRoundTrip("decimal(24, 2)", "CAST('2' AS decimal(24, 2))", createDecimalType(24, 2), "CAST('2' AS decimal(24, 2))") + .addRoundTrip("decimal(24, 2)", "CAST('2.3' AS decimal(24, 2))", createDecimalType(24, 2), "CAST('2.3' AS decimal(24, 2))") + .addRoundTrip("decimal(24, 2)", "CAST('123456789.3' AS decimal(24, 2))", createDecimalType(24, 2), "CAST('123456789.3' AS decimal(24, 2))") + .addRoundTrip("decimal(24, 4)", "CAST('12345678901234567890.31' AS decimal(24, 4))", createDecimalType(24, 4), "CAST('12345678901234567890.31' AS decimal(24, 4))") + .addRoundTrip("decimal(30, 5)", "CAST('3141592653589793238462643.38327' AS decimal(30, 5))", createDecimalType(30, 5), "CAST('3141592653589793238462643.38327' AS decimal(30, 5))") + .addRoundTrip("decimal(30, 5)", "CAST('-3141592653589793238462643.38327' AS decimal(30, 5))", createDecimalType(30, 5), "CAST('-3141592653589793238462643.38327' AS decimal(30, 5))") + .addRoundTrip("decimal(36, 0)", "CAST(NULL AS decimal(36, 0))", createDecimalType(36, 0), "CAST(NULL AS decimal(36, 0))") + .addRoundTrip("decimal(36, 0)", "CAST('999999999999999999999999999999999999' AS decimal(36, 0))", createDecimalType(36, 0), "CAST('999999999999999999999999999999999999' AS decimal(36, 0))") + .addRoundTrip("decimal(36, 0)", "CAST('-999999999999999999999999999999999999' AS decimal(36, 0))", createDecimalType(36, 0), "CAST('-999999999999999999999999999999999999' AS decimal(36, 0))") + .addRoundTrip("decimal(36, 36)", "CAST('0.27182818284590452353602874713526624977' AS decimal(36, 36))", createDecimalType(36, 36), "CAST('0.27182818284590452353602874713526624977' AS decimal(36, 36))") + .addRoundTrip("decimal(36, 36)", "CAST('-0.27182818284590452353602874713526624977' AS decimal(36, 36))", createDecimalType(36, 36), "CAST('-0.27182818284590452353602874713526624977' AS decimal(36, 36))") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_decimal")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_decimal")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_decimal")); + } + + @Test + void testDecimalDefault() + { + SqlDataTypeTest.create() + .addRoundTrip("decimal", "CAST('123456789012345.123' AS decimal(18, 3))", createDecimalType(18, 3), "CAST('123456789012345.123' AS decimal(18, 3))") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_decimal")); + } + + @Test + void testChar() + { + SqlDataTypeTest.create() + .addRoundTrip("char(10)", "NULL", VARCHAR, "CAST(NULL AS varchar)") + .addRoundTrip("char(10)", "''", VARCHAR, "CAST('' AS varchar)") + .addRoundTrip("char(6)", "'text_a'", VARCHAR, "CAST('text_a' AS varchar)") + .addRoundTrip("char(5)", "'攻殻機動隊'", VARCHAR, "CAST('攻殻機動隊' AS varchar)") + .addRoundTrip("char(1)", "'😂'", VARCHAR, "CAST('😂' AS varchar)") + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_char")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_char")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_char")); + } + + @Test + void testVarchar() + { + SqlDataTypeTest.create() + .addRoundTrip("varchar(10)", "NULL", VARCHAR, "CAST(NULL AS varchar)") + .addRoundTrip("varchar(10)", "''", VARCHAR, "CAST('' AS varchar)") + .addRoundTrip("varchar(10)", "'text_a'", VARCHAR, "CAST('text_a' AS varchar)") + .addRoundTrip("varchar(255)", "'text_b'", VARCHAR, "CAST('text_b' AS varchar)") + .addRoundTrip("varchar(65535)", "'text_d'", VARCHAR, "CAST('text_d' AS varchar)") + .addRoundTrip("varchar(5)", "'攻殻機動隊'", VARCHAR, "CAST('攻殻機動隊' AS varchar)") + .addRoundTrip("varchar(32)", "'攻殻機動隊'", VARCHAR, "CAST('攻殻機動隊' AS varchar)") + .addRoundTrip("varchar(20000)", "'攻殻機動隊'", VARCHAR, "CAST('攻殻機動隊' AS varchar)") + .addRoundTrip("varchar(1)", "'😂'", VARCHAR, "CAST('😂' AS varchar)") + .addRoundTrip("varchar(77)", "'Ну, погоди!'", VARCHAR, "CAST('Ну, погоди!' AS varchar)") + .addRoundTrip("varchar(2000000)", "'text_f'", VARCHAR, "CAST('text_f' AS varchar)") // too long for a char in Trino + .execute(getQueryRunner(), duckDbCreateAndInsert(TPCH_SCHEMA + ".test_varchar")) + .execute(getQueryRunner(), trinoCreateAsSelect(TPCH_SCHEMA + ".test_varchar")) + .execute(getQueryRunner(), trinoCreateAndInsert(TPCH_SCHEMA + ".test_varchar")); + } + + @Test + void testDate() + { + testDate(UTC); + testDate(jvmZone); + // using two non-JVM zones so that we don't need to worry what DuckDB system zone is + testDate(vilnius); + testDate(kathmandu); + testDate(TestingSession.DEFAULT_TIME_ZONE_KEY.getZoneId()); + } + + private void testDate(ZoneId sessionZone) + { + Session session = Session.builder(getSession()) + .setTimeZoneKey(TimeZoneKey.getTimeZoneKey(sessionZone.getId())) + .build(); + + SqlDataTypeTest.create() + .addRoundTrip("date", "NULL", DATE, "CAST(NULL AS DATE)") + .addRoundTrip("date", "DATE '0001-01-01'", DATE, "DATE '0001-01-01'") + .addRoundTrip("date", "DATE '1582-09-30'", DATE, "DATE '1582-09-30'") + .addRoundTrip("date", "DATE '1582-10-01'", DATE, "DATE '1582-10-01'") + .addRoundTrip("date", "DATE '1582-10-02'", DATE, "DATE '1582-10-02'") + .addRoundTrip("date", "DATE '1582-10-03'", DATE, "DATE '1582-10-03'") + .addRoundTrip("date", "DATE '1582-10-04'", DATE, "DATE '1582-10-04'") + .addRoundTrip("date", "DATE '1582-10-05'", DATE, "DATE '1582-10-05'") // Julian-Gregorian calendar cut-over + .addRoundTrip("date", "DATE '1582-10-13'", DATE, "DATE '1582-10-13'") // Julian-Gregorian calendar cut-over + .addRoundTrip("date", "DATE '1582-10-14'", DATE, "DATE '1582-10-14'") + .addRoundTrip("date", "DATE '1582-10-15'", DATE, "DATE '1582-10-15'") + .addRoundTrip("date", "DATE '1970-01-01'", DATE, "DATE '1970-01-01'") + .addRoundTrip("date", "DATE '1970-02-03'", DATE, "DATE '1970-02-03'") + .addRoundTrip("date", "DATE '2017-07-01'", DATE, "DATE '2017-07-01'") // summer on northern hemisphere (possible DST) + .addRoundTrip("date", "DATE '2017-01-01'", DATE, "DATE '2017-01-01'") // winter on northern hemisphere (possible DST on southern hemisphere) + .addRoundTrip("date", "DATE '1970-01-01'", DATE, "DATE '1970-01-01'") // change forward at midnight in JVM + .addRoundTrip("date", "DATE '1983-04-01'", DATE, "DATE '1983-04-01'") // change forward at midnight in Vilnius + .addRoundTrip("date", "DATE '1983-10-01'", DATE, "DATE '1983-10-01'") // change backward at midnight in Vilnius + .addRoundTrip("date", "DATE '9999-12-31'", DATE, "DATE '9999-12-31'") + .execute(getQueryRunner(), session, duckDbCreateAndInsert(TPCH_SCHEMA + ".test_date")) + .execute(getQueryRunner(), session, trinoCreateAsSelect(TPCH_SCHEMA + ".test_date")) + .execute(getQueryRunner(), session, trinoCreateAndInsert(TPCH_SCHEMA + ".test_date")); + } + + private DataSetup duckDbCreateAndInsert(String tableNamePrefix) + { + return new CreateAndInsertDataSetup(duckDb.getSqlExecutor(), tableNamePrefix); + } + + private DataSetup trinoCreateAsSelect(String tableNamePrefix) + { + return trinoCreateAsSelect(getSession(), tableNamePrefix); + } + + private DataSetup trinoCreateAsSelect(Session session, String tableNamePrefix) + { + return new CreateAsSelectDataSetup(new TrinoSqlExecutor(getQueryRunner(), session), tableNamePrefix); + } + + private DataSetup trinoCreateAndInsert(String tableNamePrefix) + { + return new CreateAndInsertDataSetup(new TrinoSqlExecutor(getQueryRunner()), tableNamePrefix); + } + + private static void checkIsGap(ZoneId zone, LocalDateTime dateTime) + { + verify(isGap(zone, dateTime), "Expected %s to be a gap in %s", dateTime, zone); + } + + private static boolean isGap(ZoneId zone, LocalDateTime dateTime) + { + return zone.getRules().getValidOffsets(dateTime).isEmpty(); + } + + private static void checkIsDoubled(ZoneId zone, LocalDateTime dateTime) + { + verify(zone.getRules().getValidOffsets(dateTime).size() == 2, "Expected %s to be doubled in %s", dateTime, zone); + } +} diff --git a/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestingDuckDb.java b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestingDuckDb.java new file mode 100644 index 00000000000000..7f208a985e6aef --- /dev/null +++ b/plugin/trino-duckdb/src/test/java/io/trino/plugin/duckdb/TestingDuckDb.java @@ -0,0 +1,71 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.duckdb; + +import io.trino.testing.sql.JdbcSqlExecutor; +import org.intellij.lang.annotations.Language; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class TestingDuckDb + implements Closeable +{ + public static final String TPCH_SCHEMA = "tpch"; + + private final Path path; + + public TestingDuckDb() + throws IOException + { + path = Files.createTempFile(null, null); + Files.delete(path); + } + + public String getJdbcUrl() + { + return "jdbc:duckdb:" + path.toString(); + } + + public void execute(@Language("SQL") String sql) + { + try (Connection connection = DriverManager.getConnection(getJdbcUrl(), new Properties()); + Statement statement = connection.createStatement()) { + //noinspection SqlSourceToSinkFlow + statement.execute(sql); + } + catch (SQLException e) { + throw new RuntimeException("Failed to execute statement '" + sql + "'", e); + } + } + + public JdbcSqlExecutor getSqlExecutor() + { + return new JdbcSqlExecutor(getJdbcUrl(), new Properties()); + } + + @Override + public void close() + throws IOException + { + Files.delete(path); + } +} diff --git a/pom.xml b/pom.xml index 5099f22399b366..5030e8cae6295a 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ plugin/trino-clickhouse plugin/trino-delta-lake plugin/trino-druid + plugin/trino-duckdb plugin/trino-elasticsearch plugin/trino-example-http plugin/trino-example-jdbc diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java index 92d0779735b5e6..14eb9a331f14a8 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeAllConnectors.java @@ -54,6 +54,7 @@ public void extendEnvironment(Environment.Builder builder) "clickhouse", "delta_lake", "druid", + "duckdb", "elasticsearch", "gsheets", "hive", diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-all/duckdb.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-all/duckdb.properties new file mode 100644 index 00000000000000..8e23c0b5e7d704 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/presto-product-tests/conf/environment/multinode-all/duckdb.properties @@ -0,0 +1,2 @@ +connector.name=duckdb +connection-url=jdbc:duckdb://