From e7293f09197e5e10c98a0c341990758da1bc9b08 Mon Sep 17 00:00:00 2001 From: hbc Date: Sun, 26 Feb 2023 15:44:56 -0800 Subject: [PATCH 1/9] test: add test case for sqlite datetime --- prql-compiler/src/sql/gen_query.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/prql-compiler/src/sql/gen_query.rs b/prql-compiler/src/sql/gen_query.rs index fba04d0be692..5eb6189201f4 100644 --- a/prql-compiler/src/sql/gen_query.rs +++ b/prql-compiler/src/sql/gen_query.rs @@ -846,4 +846,26 @@ mod test { _expr_0 > 3 "###); } + + #[test] + fn test_sqlite_datetime() { + let query = &r#" + from test_table + select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] + "#; + + let opts = crate::Options::default() + .no_signature() + .with_target(crate::Target::Sql(Some(crate::sql::Dialect::SQLite))); + + assert_snapshot!( + crate::compile(query, &opts).unwrap(), + @r###"SELECT + DATE('2022-12-31') AS date, + TIME('08:30') AS time, + DATETIME('2020-01-01T13:19:55-0800') AS timestamp +FROM + test_tables"### + ); + } } From 5370fc8d4e7dee23295b761cab34ead005edab95 Mon Sep 17 00:00:00 2001 From: hbc Date: Sun, 26 Feb 2023 17:21:30 -0800 Subject: [PATCH 2/9] feat: implement `is` helper for DialectHandler --- prql-compiler/src/sql/dialect.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/prql-compiler/src/sql/dialect.rs b/prql-compiler/src/sql/dialect.rs index 6bc52b2859c7..18a74ef5e729 100644 --- a/prql-compiler/src/sql/dialect.rs +++ b/prql-compiler/src/sql/dialect.rs @@ -14,6 +14,7 @@ use core::fmt::Debug; use serde::{Deserialize, Serialize}; +use std::any::{Any, TypeId}; use strum::{EnumMessage, IntoEnumIterator}; /// SQL dialect. @@ -109,7 +110,7 @@ pub(super) enum ColumnExclude { Except, } -pub(super) trait DialectHandler { +pub(super) trait DialectHandler: Any { fn use_top(&self) -> bool { false } @@ -155,6 +156,13 @@ pub(super) trait DialectHandler { } } +impl dyn DialectHandler { + #[inline] + pub fn is(&self) -> bool { + TypeId::of::() == self.type_id() + } +} + impl DialectHandler for GenericDialect {} impl DialectHandler for PostgresDialect { From 60575ca7b2fa4b8172cfb6a4fe23eb213ca03a80 Mon Sep 17 00:00:00 2001 From: hbc Date: Sun, 26 Feb 2023 17:22:16 -0800 Subject: [PATCH 3/9] fix: use datetime functions for sqlite dialect --- prql-compiler/src/sql/gen_expr.rs | 213 +++++++++++++++++++++++++++-- prql-compiler/src/sql/gen_query.rs | 26 +++- 2 files changed, 226 insertions(+), 13 deletions(-) diff --git a/prql-compiler/src/sql/gen_expr.rs b/prql-compiler/src/sql/gen_expr.rs index 271b4a6a1833..c5eb71134348 100644 --- a/prql-compiler/src/sql/gen_expr.rs +++ b/prql-compiler/src/sql/gen_expr.rs @@ -143,18 +143,17 @@ pub(super) fn translate_literal(l: Literal, ctx: &Context) -> Result sql_ast::Expr::Value(Value::Boolean(b)), Literal::Float(f) => sql_ast::Expr::Value(Value::Number(format!("{f:?}"), false)), Literal::Integer(i) => sql_ast::Expr::Value(Value::Number(format!("{i}"), false)), - Literal::Date(value) => sql_ast::Expr::TypedString { - data_type: sql_ast::DataType::Date, + Literal::Date(value) => translate_datetime_literal(sql_ast::DataType::Date, value, ctx), + Literal::Time(value) => translate_datetime_literal( + sql_ast::DataType::Time(None, sql_ast::TimezoneInfo::None), value, - }, - Literal::Time(value) => sql_ast::Expr::TypedString { - data_type: sql_ast::DataType::Time(None, sql_ast::TimezoneInfo::None), - value, - }, - Literal::Timestamp(value) => sql_ast::Expr::TypedString { - data_type: sql_ast::DataType::Timestamp(None, sql_ast::TimezoneInfo::None), + ctx, + ), + Literal::Timestamp(value) => translate_datetime_literal( + sql_ast::DataType::Timestamp(None, sql_ast::TimezoneInfo::None), value, - }, + ctx, + ), Literal::ValueAndUnit(vau) => { let sql_parser_datetime = match vau.unit.as_str() { "years" => DateTimeField::Year, @@ -184,6 +183,63 @@ pub(super) fn translate_literal(l: Literal, ctx: &Context) -> Result sql_ast::Expr { + if ctx.dialect.is::() { + translate_datetime_literal_with_sqlite_function(data_type, value) + } else { + translate_dateatime_literal_with_typed_string(data_type, value) + } +} + +fn translate_dateatime_literal_with_typed_string( + data_type: sql_ast::DataType, + value: String, +) -> sql_ast::Expr { + sql_ast::Expr::TypedString { data_type, value } +} + +fn translate_datetime_literal_with_sqlite_function( + data_type: sql_ast::DataType, + value: String, +) -> sql_ast::Expr { + let timezone_indicator_regex = Regex::new(r"([+-]\d{2}):?(\d{2})$").unwrap(); + let time_value = if let Some(groups) = timezone_indicator_regex.captures(value.as_str()) { + // formalize the timezone indicator to be [+-]HH:MM + // ref: https://www.sqlite.org/lang_datefunc.html + timezone_indicator_regex + .replace( + &value, + format!("{}:{}", groups[1].to_string(), groups[2].to_string()).as_str(), + ) + .to_string() + } else { + value + }; + + let arg = FunctionArg::Unnamed(FunctionArgExpr::Expr(sql_ast::Expr::Value( + Value::SingleQuotedString(time_value), + ))); + + let func_name = match data_type { + sql_ast::DataType::Date => data_type.to_string(), + sql_ast::DataType::Time(..) => data_type.to_string(), + sql_ast::DataType::Timestamp(..) => "DATETIME".to_string(), + _ => unreachable!(), + }; + + sql_ast::Expr::Function(Function { + name: ObjectName(vec![sql_ast::Ident::new(func_name)]), + args: vec![arg], + over: None, + distinct: false, + special: false, + }) +} + pub(super) fn translate_cid(cid: CId, ctx: &mut Context) -> Result { if ctx.query.pre_projection { log::debug!("translating {cid:?} pre projection"); @@ -930,4 +986,141 @@ mod test { Ok(()) } + + #[test] + fn test_translate_datetime_literal_with_sqlite_function() -> Result<()> { + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Date, + "2020-01-01".to_string(), + ), + @r###" +--- +Function: + name: + - value: DATE + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: 2020-01-01 + over: ~ + distinct: false + special: false +"### + ); + + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Time(None, sql_ast::TimezoneInfo::None), + "03:05".to_string(), + ), + @r###" +--- +Function: + name: + - value: TIME + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: "03:05" + over: ~ + distinct: false + special: false +"### + ); + + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Time(None, sql_ast::TimezoneInfo::None), + "03:05+08:00".to_string(), + ), + @r###" +--- +Function: + name: + - value: TIME + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: "03:05+08:00" + over: ~ + distinct: false + special: false +"### + ); + + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Time(None, sql_ast::TimezoneInfo::None), + "03:05+0800".to_string(), + ), + @r###" +--- +Function: + name: + - value: TIME + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: "03:05+08:00" + over: ~ + distinct: false + special: false +"### + ); + + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Timestamp(None, sql_ast::TimezoneInfo::None), + "2021-03-14T03:05+0800".to_string(), + ), + @r###" +--- +Function: + name: + - value: DATETIME + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: "2021-03-14T03:05+08:00" + over: ~ + distinct: false + special: false +"### + ); + + assert_yaml_snapshot!( + translate_datetime_literal_with_sqlite_function( + sql_ast::DataType::Timestamp(None, sql_ast::TimezoneInfo::None), + "2021-03-14T03:05+08:00".to_string(), + ), + @r###" +--- +Function: + name: + - value: DATETIME + quote_style: ~ + args: + - Unnamed: + Expr: + Value: + SingleQuotedString: "2021-03-14T03:05+08:00" + over: ~ + distinct: false + special: false +"### + ); + + Ok(()) + } } diff --git a/prql-compiler/src/sql/gen_query.rs b/prql-compiler/src/sql/gen_query.rs index 5eb6189201f4..f1449391bf75 100644 --- a/prql-compiler/src/sql/gen_query.rs +++ b/prql-compiler/src/sql/gen_query.rs @@ -848,7 +848,26 @@ mod test { } #[test] - fn test_sqlite_datetime() { + fn test_datetime() { + let query = &r#" + from test_table + select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] + "#; + + assert_snapshot!( + crate::test::compile(query).unwrap(), + @r###"SELECT + DATE '2022-12-31' AS date, + TIME '08:30' AS time, + TIMESTAMP '2020-01-01T13:19:55-0800' AS timestamp +FROM + test_table +"### + ) + } + + #[test] + fn test_datetime_sqlite() { let query = &r#" from test_table select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] @@ -863,9 +882,10 @@ mod test { @r###"SELECT DATE('2022-12-31') AS date, TIME('08:30') AS time, - DATETIME('2020-01-01T13:19:55-0800') AS timestamp + DATETIME('2020-01-01T13:19:55-08:00') AS timestamp FROM - test_tables"### + test_table +"### ); } } From dcd8a07f69ce20920c34e363fc90719b3328653e Mon Sep 17 00:00:00 2001 From: hbc Date: Sun, 26 Feb 2023 17:29:39 -0800 Subject: [PATCH 4/9] style: fix lint issues --- prql-compiler/src/sql/gen_expr.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/prql-compiler/src/sql/gen_expr.rs b/prql-compiler/src/sql/gen_expr.rs index c5eb71134348..a214a48a4e80 100644 --- a/prql-compiler/src/sql/gen_expr.rs +++ b/prql-compiler/src/sql/gen_expr.rs @@ -211,10 +211,7 @@ fn translate_datetime_literal_with_sqlite_function( // formalize the timezone indicator to be [+-]HH:MM // ref: https://www.sqlite.org/lang_datefunc.html timezone_indicator_regex - .replace( - &value, - format!("{}:{}", groups[1].to_string(), groups[2].to_string()).as_str(), - ) + .replace(&value, format!("{}:{}", &groups[1], &groups[2]).as_str()) .to_string() } else { value From f124b7e020bc7560830bff8679f4a0ceefaed40c Mon Sep 17 00:00:00 2001 From: hbc Date: Mon, 27 Feb 2023 14:36:17 -0800 Subject: [PATCH 5/9] Update prql-compiler/src/sql/gen_expr.rs Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- prql-compiler/src/sql/gen_expr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prql-compiler/src/sql/gen_expr.rs b/prql-compiler/src/sql/gen_expr.rs index a214a48a4e80..6afebb837cd1 100644 --- a/prql-compiler/src/sql/gen_expr.rs +++ b/prql-compiler/src/sql/gen_expr.rs @@ -195,7 +195,7 @@ fn translate_datetime_literal( } } -fn translate_dateatime_literal_with_typed_string( +fn translate_datetime_literal_with_typed_string( data_type: sql_ast::DataType, value: String, ) -> sql_ast::Expr { From 0f9f164cf3af09a6127ce12b2c10aac0221a1b24 Mon Sep 17 00:00:00 2001 From: hbc Date: Mon, 27 Feb 2023 14:36:25 -0800 Subject: [PATCH 6/9] Update prql-compiler/src/sql/gen_expr.rs Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- prql-compiler/src/sql/gen_expr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prql-compiler/src/sql/gen_expr.rs b/prql-compiler/src/sql/gen_expr.rs index 6afebb837cd1..d523a3b6428f 100644 --- a/prql-compiler/src/sql/gen_expr.rs +++ b/prql-compiler/src/sql/gen_expr.rs @@ -191,7 +191,7 @@ fn translate_datetime_literal( if ctx.dialect.is::() { translate_datetime_literal_with_sqlite_function(data_type, value) } else { - translate_dateatime_literal_with_typed_string(data_type, value) + translate_datetime_literal_with_typed_string(data_type, value) } } From 4660de2aadb45b49e823763d5a3bdc0fe4f9831a Mon Sep 17 00:00:00 2001 From: hbc Date: Mon, 27 Feb 2023 14:36:46 -0800 Subject: [PATCH 7/9] Update prql-compiler/src/sql/gen_expr.rs Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- prql-compiler/src/sql/gen_expr.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/prql-compiler/src/sql/gen_expr.rs b/prql-compiler/src/sql/gen_expr.rs index d523a3b6428f..7f17c4e9fe88 100644 --- a/prql-compiler/src/sql/gen_expr.rs +++ b/prql-compiler/src/sql/gen_expr.rs @@ -206,6 +206,8 @@ fn translate_datetime_literal_with_sqlite_function( data_type: sql_ast::DataType, value: String, ) -> sql_ast::Expr { + // TODO: promote parsing timezone handling to the parser; we should be storing + // structured data rather than strings in the AST let timezone_indicator_regex = Regex::new(r"([+-]\d{2}):?(\d{2})$").unwrap(); let time_value = if let Some(groups) = timezone_indicator_regex.captures(value.as_str()) { // formalize the timezone indicator to be [+-]HH:MM From 9545cdecf3a66bfba83c16388d9570e690be1058 Mon Sep 17 00:00:00 2001 From: hbc Date: Mon, 27 Feb 2023 14:42:59 -0800 Subject: [PATCH 8/9] test: move tests to test.rs --- prql-compiler/src/sql/gen_query.rs | 42 ---------------------------- prql-compiler/src/test.rs | 44 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/prql-compiler/src/sql/gen_query.rs b/prql-compiler/src/sql/gen_query.rs index f1449391bf75..fba04d0be692 100644 --- a/prql-compiler/src/sql/gen_query.rs +++ b/prql-compiler/src/sql/gen_query.rs @@ -846,46 +846,4 @@ mod test { _expr_0 > 3 "###); } - - #[test] - fn test_datetime() { - let query = &r#" - from test_table - select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] - "#; - - assert_snapshot!( - crate::test::compile(query).unwrap(), - @r###"SELECT - DATE '2022-12-31' AS date, - TIME '08:30' AS time, - TIMESTAMP '2020-01-01T13:19:55-0800' AS timestamp -FROM - test_table -"### - ) - } - - #[test] - fn test_datetime_sqlite() { - let query = &r#" - from test_table - select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] - "#; - - let opts = crate::Options::default() - .no_signature() - .with_target(crate::Target::Sql(Some(crate::sql::Dialect::SQLite))); - - assert_snapshot!( - crate::compile(query, &opts).unwrap(), - @r###"SELECT - DATE('2022-12-31') AS date, - TIME('08:30') AS time, - DATETIME('2020-01-01T13:19:55-08:00') AS timestamp -FROM - test_table -"### - ); - } } diff --git a/prql-compiler/src/test.rs b/prql-compiler/src/test.rs index 7fe43edc3c11..6ee196ba0d0b 100644 --- a/prql-compiler/src/test.rs +++ b/prql-compiler/src/test.rs @@ -3318,5 +3318,49 @@ fn test_params() { ) AND total > $3 "### + ) + } + +// for #1969 +#[test] +fn test_datetime() { + let query = &r#" + from test_table + select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] + "#; + + assert_snapshot!( + compile(query).unwrap(), + @r###"SELECT + DATE '2022-12-31' AS date, + TIME '08:30' AS time, + TIMESTAMP '2020-01-01T13:19:55-0800' AS timestamp +FROM + test_table +"### + ) +} + +// for #1969 +#[test] +fn test_datetime_sqlite() { + let query = &r#" + from test_table + select [date = @2022-12-31, time = @08:30, timestamp = @2020-01-01T13:19:55-0800] + "#; + + let opts = Options::default() + .no_signature() + .with_target(Target::Sql(Some(sql::Dialect::SQLite))); + + assert_snapshot!( + crate::compile(query, &opts).unwrap(), + @r###"SELECT + DATE('2022-12-31') AS date, + TIME('08:30') AS time, + DATETIME('2020-01-01T13:19:55-08:00') AS timestamp +FROM + test_table +"### ); } From b8d3c5cf7ae234556376b1b785b9050ec812ec0b Mon Sep 17 00:00:00 2001 From: hbc Date: Mon, 27 Feb 2023 15:56:52 -0800 Subject: [PATCH 9/9] style: lint fix --- prql-compiler/src/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prql-compiler/src/test.rs b/prql-compiler/src/test.rs index 6ee196ba0d0b..0b27d95119a8 100644 --- a/prql-compiler/src/test.rs +++ b/prql-compiler/src/test.rs @@ -3319,7 +3319,7 @@ fn test_params() { AND total > $3 "### ) - } +} // for #1969 #[test]