From e1922bfa7873bb6ef45c7f6b9956baf185491d20 Mon Sep 17 00:00:00 2001 From: hemanth94 Date: Tue, 13 Feb 2024 03:49:31 +0530 Subject: [PATCH 1/2] Unsigned integers issue --- crates/datasources/src/native/access.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/datasources/src/native/access.rs b/crates/datasources/src/native/access.rs index 4e5a23b8d..c813a4c24 100644 --- a/crates/datasources/src/native/access.rs +++ b/crates/datasources/src/native/access.rs @@ -120,6 +120,17 @@ fn arrow_to_delta_safe(arrow_type: &DataType) -> DeltaResult { metadata: Some(metadata), }) } + dtype @ (DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64) => { + let mut metadata = HashMap::new(); + metadata.insert("arrow_type".to_string(), json!(dtype)); + + let delta_type = dtype.try_into()?; + + Ok(DeltaField { + data_type: delta_type, + metadata: Some(metadata), + }) + } other => { let delta_type = other.try_into()?; Ok(DeltaField { @@ -130,7 +141,6 @@ fn arrow_to_delta_safe(arrow_type: &DataType) -> DeltaResult { } } - impl NativeTableStorage { /// Create a native table storage provider from a URL and an object store instance /// rooted at that location. @@ -194,7 +204,6 @@ impl NativeTableStorage { .with_table_name(&table.meta.name) .with_log_store(delta_store); - for col in &opts.columns { let delta_col = arrow_to_delta_safe(&col.arrow_type)?; @@ -371,7 +380,6 @@ impl TableProvider for NativeTable { if let Some(arrow_type) = metadata.get("arrow_type") { // this is dumb AF, delta-lake is returning a string of a json object instead of a json object - // any panics here are bugs in writing the metadata in the first place let s: String = serde_json::from_str(arrow_type).expect("metadata was not correctly written"); From 46ef826de2ffecd6104d1e773e5222fdc1fd60fc Mon Sep 17 00:00:00 2001 From: hemanth94 Date: Tue, 13 Feb 2024 21:17:18 +0530 Subject: [PATCH 2/2] SLT for 2588 --- testdata/sqllogictests/datatypes.slt | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/testdata/sqllogictests/datatypes.slt b/testdata/sqllogictests/datatypes.slt index cef57cd9b..25e78dd6f 100644 --- a/testdata/sqllogictests/datatypes.slt +++ b/testdata/sqllogictests/datatypes.slt @@ -33,4 +33,20 @@ statement error select array_element(1, f) from test; statement error -select array_append(f, 3) from test; \ No newline at end of file +select array_append(f, 3) from test; + + + +# 2588 Unsigned integers downgraded to integers in delta-lake +statement ok +create table datatypes (u8 tinyint unsigned, u16 smallint unsigned, u32 int unsigned, u64 bigint unsigned); + + +# Inserting negative values to test validity of unsigned integers +statement error +insert into datatypes (u8, u16, u32, u64) values (-1, -1, -1, -1); + + +# Inserting positive values +statement ok +insert into datatypes (u8, u16, u32, u64) values (1, 1, 1, 1); \ No newline at end of file