Merge pull request #282 from Qrlew/add_parsing_values

add support for extract_epoch function

CHANGELOG.md

@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.9.20] - 2024-06-20
+### Added
+- support for extract_epoch function
+- support for NULL and Boolean values in the SELECT when parsing a query 
+
+### Changed
+- Polymorphic properties of count(distinct ) are set as those of count
+
 ## [0.9.19] - 2024-05-23
 ### Added
 - parsing of CTEs with column alias

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Nicolas Grislain <ng@sarus.tech>"]
 name = "qrlew"
-version = "0.9.19"
+version = "0.9.20"
 edition = "2021"
 description = "Sarus Qrlew Engine"
 documentation = "https://docs.rs/qrlew"

src/data_type/function.rs

@@ -1944,6 +1944,20 @@ pub fn extract_year() -> impl Function {
     ))
 }
 
+pub fn extract_epoch() -> impl Function {
+    Polymorphic::default()
+        .with(PartitionnedMonotonic::univariate(
+            data_type::DateTime::default(),
+            |a| {
+                (a.and_utc().timestamp() as i64).clamp(<i64 as Bound>::min(), <i64 as Bound>::max())
+            },
+        ))
+        .with(PartitionnedMonotonic::univariate(
+            data_type::Duration::default(),
+            |a| (a.num_seconds()).clamp(<i64 as Bound>::min(), <i64 as Bound>::max()),
+        ))
+}
+
 pub fn extract_month() -> impl Function {
     Polymorphic::from((
         Pointwise::univariate(
@@ -2449,7 +2463,7 @@ pub fn count_distinct() -> impl Function {
         Aggregate::from(
             DataType::Any,
             |values| (values.iter().cloned().collect::<HashSet<_>>().len() as i64).into(),
-            |(_dt, size)| Ok(data_type::Integer::from_interval(1, *size.max().unwrap())),
+            |(_dt, size)| Ok(size),
         ),
         // Optional implementation
         Aggregate::from(
@@ -4316,6 +4330,37 @@ mod tests {
 
     #[test]
     fn test_extract() {
+        // epoch
+        println!("\nTest extract_epoch");
+        let fun = extract_epoch();
+        println!("type = {}", fun);
+        println!("domain = {}", fun.domain());
+        println!("co_domain = {}", fun.co_domain());
+        println!("data_type = {}", fun.data_type());
+
+        let set = DataType::date_time_values([
+            NaiveDate::from_ymd_opt(2016, 7, 8)
+                .unwrap()
+                .and_hms_opt(9, 10, 11)
+                .unwrap(),
+            NaiveDate::from_ymd_opt(2026, 7, 8)
+                .unwrap()
+                .and_hms_opt(9, 15, 11)
+                .unwrap(),
+        ]);
+
+        let im = fun.super_image(&set).unwrap();
+        println!("im({}) = {}", set, im);
+        assert!(im == DataType::integer_values([1467969011, 1783502111]));
+
+        let set = DataType::duration_values([
+            chrono::Duration::hours(24),
+            chrono::Duration::seconds(100),
+        ]);
+        let im = fun.super_image(&set).unwrap();
+        println!("im({}) = {}", set, im);
+        assert!(im == DataType::integer_values([86400, 100]));
+
         // year
         println!("\nTest extract_year");
         let fun = extract_year();

src/dialect_translation/mod.rs

@@ -58,6 +58,20 @@ macro_rules! unary_function_ast_constructor {
     }
 }
 
+/// Constructors for creating trait functions with default implementations for generating AST extract expressions
+macro_rules! extract_ast_expression_constructor {
+    ($( $enum:ident ),*) => {
+        paste! {
+            $(
+                fn [<extract_ $enum:snake>](&self, expr: &expr::Expr) -> ast::Expr {
+                    let ast_expr = self.expr(expr);
+                    extract_builder(ast_expr, ast::DateTimeField::$enum)
+                }
+            )*
+        }
+    }
+}
+
 /// Constructors for creating trait functions with default implementations for generating AST nary function expressions
 macro_rules! nary_function_ast_constructor {
     ($( $enum:ident ),*) => {
@@ -362,7 +376,10 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                     expr::Value::Bytes(_) => todo!(),
                     expr::Value::Struct(_) => todo!(),
                     expr::Value::Union(_) => todo!(),
-                    expr::Value::Optional(_) => todo!(),
+                    expr::Value::Optional(optional_val) => match optional_val.as_deref() {
+                        Some(arg) => self.value(arg),
+                        None => ast::Expr::Value(ast::Value::Null),
+                    },
                     expr::Value::List(l) => ast::Expr::Tuple(
                         l.to_vec()
                             .iter()
@@ -435,6 +452,7 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                         CastAsTime,
                         Sign,
                         Unhex,
+                        ExtractEpoch,
                         ExtractYear,
                         ExtractMonth,
                         ExtractDay,
@@ -554,16 +572,6 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                 CastAsTime,
                 Sign,
                 Unhex,
-                ExtractYear,
-                ExtractMonth,
-                ExtractDay,
-                ExtractHour,
-                ExtractMinute,
-                ExtractSecond,
-                ExtractMicrosecond,
-                ExtractMillisecond,
-                ExtractDow,
-                ExtractWeek,
                 Dayname,
                 UnixTimestamp,
                 Quarter,
@@ -607,6 +615,24 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                 Concat
             );
 
+            extract_ast_expression_constructor!(
+                Epoch,
+                Year,
+                Month,
+                Day,
+                Dow,
+                Hour,
+                Minute,
+                Second,
+                Microsecond,
+                Millisecond
+            );
+
+            fn extract_week(&self, expr: &expr::Expr) -> ast::Expr {
+                let ast_expr = self.expr(expr);
+                extract_builder(ast_expr, ast::DateTimeField::Week(None))
+            }
+
             fn cast_as_text(&self, expr: &expr::Expr) -> ast::Expr {
                 let ast_expr = self.expr(expr);
                 cast_builder(ast_expr, ast::DataType::Text)
@@ -870,6 +896,13 @@ fn unary_op_builder(op: ast::UnaryOperator, expr: ast::Expr) -> ast::Expr {
     }
 }
 
+fn extract_builder(expr: ast::Expr, datetime_field: ast::DateTimeField) -> ast::Expr {
+    ast::Expr::Extract {
+        field: datetime_field,
+        expr: Box::new(expr),
+    }
+}
+
 pub struct RelationWithTranslator<'a, T: RelationToQueryTranslator>(pub &'a Relation, pub T);
 
 impl<'a, T: RelationToQueryTranslator> From<RelationWithTranslator<'a, T>> for ast::Query {

src/dialect_translation/mssql.rs

@@ -135,10 +135,15 @@ impl RelationToQueryTranslator for MsSqlTranslator {
         function_builder("CEILING", vec![arg], false)
     }
 
-    // fn from_extract_epoch(&self, expr: &expr::Expr) -> ast::Expr {
-    //     //EXTRACT(EPOCH FROM col1) is not supported yet
-    //     todo!()
-    // }
+    fn extract_epoch(&self, expr: &expr::Expr) -> ast::Expr {
+        let arg = self.expr(expr);
+        let second = ast::Expr::Identifier(ast::Ident {
+            value: "SECOND".to_string(),
+            quote_style: None,
+        });
+        let unix = ast::Expr::Value(ast::Value::SingleQuotedString("19700101".to_string()));
+        function_builder("DATEDIFF", vec![second, unix, arg], false)
+    }
 
     // used during onboarding in order to have datetime with a proper format.
     // This is not needed when we will remove the cast in string of the datetime

src/dialect_translation/postgresql.rs

@@ -169,4 +169,46 @@ mod tests {
             .map(ToString::to_string);
         Ok(())
     }
+
+    #[test]
+    fn test_relation_to_query_with_null_field() -> Result<()> {
+        let mut database = postgresql::test_database();
+        let relations = database.relations();
+        let query_str = r#"SELECT CASE WHEN (1) > (2) THEN 1 ELSE NULL END AS "_PRIVACY_UNIT_", a AS a FROM table_1"#;
+        let translator = PostgreSqlTranslator;
+        let query = parse_with_dialect(query_str, translator.dialect())?;
+        let query_with_relation = QueryWithRelations::new(&query, &relations);
+        let relation = Relation::try_from((query_with_relation, translator))?;
+        println!("\n {} \n", relation);
+        let rel_with_traslator = RelationWithTranslator(&relation, translator);
+        let translated = ast::Query::from(rel_with_traslator);
+        print!("{}", translated);
+        _ = database
+            .query(translated.to_string().as_str())
+            .unwrap()
+            .iter()
+            .map(ToString::to_string);
+        Ok(())
+    }
+
+    #[test]
+    fn test_extract() -> Result<()> {
+        let mut database = postgresql::test_database();
+        let relations = database.relations();
+        let query_str = r#"SELECT extract(EPOCH FROM c) as my_epoch, extract(YEAR FROM c) as my_year, extract(WEEK FROM c) as my_week, extract(DOW FROM c) as my_dow FROM table_1"#;
+        let translator = PostgreSqlTranslator;
+        let query = parse_with_dialect(query_str, translator.dialect())?;
+        let query_with_relation = QueryWithRelations::new(&query, &relations);
+        let relation = Relation::try_from((query_with_relation, translator))?;
+        println!("\n {} \n", relation);
+        let rel_with_traslator = RelationWithTranslator(&relation, translator);
+        let translated = ast::Query::from(rel_with_traslator);
+        print!("{}", translated);
+        _ = database
+            .query(translated.to_string().as_str())
+            .unwrap()
+            .iter()
+            .map(ToString::to_string);
+        Ok(())
+    }
 }

src/expr/function.rs

@@ -76,6 +76,7 @@ pub enum Function {
     CurrentDate,
     CurrentTime,
     CurrentTimestamp,
+    ExtractEpoch,
     ExtractYear,
     ExtractMonth,
     ExtractDay,
@@ -173,6 +174,7 @@ impl Function {
             | Function::CastAsTime
             | Function::Sign
             | Function::Unhex
+            | Function::ExtractEpoch
             | Function::ExtractYear
             | Function::ExtractMonth
             | Function::ExtractDay
@@ -276,6 +278,7 @@ impl Function {
             | Function::Floor
             | Function::Sign
             | Function::Unhex
+            | Function::ExtractEpoch
             | Function::ExtractYear
             | Function::ExtractMonth
             | Function::ExtractDay
@@ -431,6 +434,7 @@ impl fmt::Display for Function {
             Function::CastAsTime => "cast_as_time",
             Function::Sign => "sign",
             Function::Unhex => "unhex",
+            Function::ExtractEpoch => "extract_epoch",
             Function::ExtractYear => "extract_year",
             Function::ExtractMonth => "extract_month",
             Function::ExtractDay => "extract_day",

src/expr/implementation.rs

@@ -109,6 +109,7 @@ function_implementations!(
         RegexpContains,
         Encode,
         Decode,
+        ExtractEpoch,
         ExtractYear,
         ExtractMonth,
         ExtractDay,

src/expr/mod.rs

@@ -317,6 +317,7 @@ impl_unary_function_constructors!(
     Floor,
     Sign,
     Unhex,
+    ExtractEpoch,
     ExtractYear,
     ExtractMonth,
     ExtractDay,

src/expr/sql.rs

@@ -337,6 +337,10 @@ impl<'a> expr::Visitor<'a, ast::Expr> for FromExprVisitor {
                 null_treatment: None,
                 within_group: vec![],
             }),
+            expr::function::Function::ExtractEpoch => ast::Expr::Extract {
+                field: ast::DateTimeField::Epoch,
+                expr: arguments[0].clone().into(),
+            },
             expr::function::Function::ExtractYear => ast::Expr::Extract {
                 field: ast::DateTimeField::Year,
                 expr: arguments[0].clone().into(),
@@ -583,7 +587,7 @@ impl From<&Expr> for ast::Expr {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::sql::parse_expr;
+    use crate::{hierarchy::Hierarchy, sql::parse_expr, WithoutContext};
     use std::convert::TryFrom;
 
     #[test]
@@ -1209,6 +1213,15 @@ mod tests {
 
     #[test]
     fn test_extract() {
+        // EXTRACT(EPOCH FROM col1)
+        let str_expr = "extract(epoch from col1)";
+        let ast_expr: ast::Expr = parse_expr(str_expr).unwrap();
+        let expr = Expr::try_from(&ast_expr).unwrap();
+        println!("expr = {}", expr);
+        let gen_expr = ast::Expr::from(&expr);
+        println!("ast::expr = {gen_expr}");
+        assert_eq!(gen_expr, ast_expr);
+
         // EXTRACT(YEAR FROM col1)
         let str_expr = "extract(year from col1)";
         let ast_expr: ast::Expr = parse_expr(str_expr).unwrap();
@@ -1488,4 +1501,27 @@ mod tests {
         println!("ast::expr = {gen_expr}");
         assert_eq!(gen_expr, ast_expr);
     }
+
+    #[test]
+    fn test_expr_from_value() {
+        let ast_expr: ast::Expr = parse_expr("True").unwrap();
+        println!("\nast::expr = {ast_expr}");
+        let expr = Expr::try_from(ast_expr.with(&Hierarchy::empty())).unwrap();
+        println!("expr = {}", expr);
+
+        let ast_expr: ast::Expr = parse_expr("1").unwrap();
+        println!("\nast::expr = {ast_expr}");
+        let expr = Expr::try_from(ast_expr.with(&Hierarchy::empty())).unwrap();
+        println!("expr = {}", expr);
+
+        let ast_expr: ast::Expr = parse_expr("Null").unwrap();
+        println!("\nast::expr = {ast_expr}");
+        let expr = Expr::try_from(ast_expr.with(&Hierarchy::empty())).unwrap();
+        println!("expr = {}", expr);
+
+        let ast_expr: ast::Expr = parse_expr(" 'some_string' ").unwrap();
+        println!("\nast::expr = {ast_expr}");
+        let expr = Expr::try_from(ast_expr.with(&Hierarchy::empty())).unwrap();
+        println!("expr = {}", expr);
+    }
 }

src/rewriting/composition.rs

@@ -261,6 +261,10 @@ mod tests {
             ("b", DataType::float_interval(-2., 2.)),
             ("c", DataType::float()),
             ("d", DataType::float_interval(0., 1.)),
+            ("e", DataType::float()),
+            ("f", DataType::float_interval(-2., 2.)),
+            ("g", DataType::float()),
+            ("h", DataType::float_interval(0., 1.)),
         ]
         .into_iter()
         .collect();