Merge pull request #239 from Qrlew/bigquery_connector

Add big query connector for integration tests

.github/workflows/ci.yml

@@ -51,6 +51,6 @@ jobs:
         steps:
             - uses: actions/checkout@v3
             - name: Build
-              run: cargo build --features mssql --verbose
+              run: cargo build --features mssql,bigquery --verbose
             - name: Run tests
-              run: cargo test --features mssql --verbose
+              run: cargo test --features mssql,bigquery --verbose

.gitignore

@@ -15,7 +15,7 @@ Cargo.lock
 
 
 # Added by cargo
-
 /target
 
+# Do not commit vscode settings
 .vscode/

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Changed
 
+## [0.9.7] - 2024-01-24
+### Added
+- support for bigquery, connector and translator [#239](https://github.com/Qrlew/qrlew/pull/239)
+
 ## [0.9.5] - 2024-01-18
 ### Fixed
 - bug mssql translator [#254](https://github.com/Qrlew/qrlew/pull/254)

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 authors = ["Nicolas Grislain <ng@sarus.tech>"]
 name = "qrlew"
-version = "0.9.6"
+version = "0.9.7"
 edition = "2021"
 description = "Sarus Qrlew Engine"
 documentation = "https://docs.rs/qrlew"
@@ -36,10 +36,16 @@ statrs = "0.16.0"
 sqlx = { version = "0.6.3", features = ["mssql", "runtime-tokio-native-tls", "offline", "any"], optional = true }
 tokio = { version = "1", features = ["full"], optional = true }
 
+# bigquery dependencies
+gcp-bigquery-client = { version = "0.18", optional = true }
+wiremock = { version = "0.5.19", optional = true }
+tempfile = { version = "3.6.0", optional = true }
+
 [features]
 # Use SQLite for tests and examples
 sqlite = ["dep:rusqlite"]
 mssql = ["dep:sqlx", "dep:tokio"]
+bigquery = ["dep:gcp-bigquery-client", "dep:wiremock", "dep:tempfile"]
 # Tests
 checked_injections = []
 # Multiplicity features are tested on large datasets (may take too much memory)

src/dialect_translation/bigquery.rs

@@ -1 +1,155 @@
+use crate::{
+    expr::{self},
+    relation::{Join, Variant as _},
+};
 
+use super::{function_builder, QueryToRelationTranslator, RelationToQueryTranslator};
+use sqlparser::{ast, dialect::BigQueryDialect};
+
+
+
+#[derive(Clone, Copy)]
+pub struct BigQueryTranslator;
+
+impl RelationToQueryTranslator for BigQueryTranslator {
+    fn cte(&self, name: ast::Ident, _columns: Vec<ast::Ident>, query: ast::Query) -> ast::Cte {
+        ast::Cte {
+            alias: ast::TableAlias {
+                name,
+                columns: vec![],
+            },
+            query: Box::new(query),
+            from: None,
+        }
+    }
+    fn first(&self, expr: &expr::Expr) -> ast::Expr {
+        ast::Expr::from(expr)
+    }
+
+    fn mean(&self, expr: &expr::Expr) -> ast::Expr {
+        let arg = self.expr(expr);
+        function_builder("AVG", vec![arg], false)
+    }
+
+    fn var(&self, expr: &expr::Expr) -> ast::Expr {
+        let arg = self.expr(expr);
+        function_builder("VARIANCE", vec![arg], false)
+    }
+
+    fn std(&self, expr: &expr::Expr) -> ast::Expr {
+        let arg = self.expr(expr);
+        function_builder("STDDEV", vec![arg], false)
+    }
+    /// Converting LOG to LOG10
+    fn log(&self,expr: &expr::Expr) -> ast::Expr {
+        let arg = self.expr(expr);
+        function_builder("LOG10", vec![arg], false)
+    }
+    fn cast_as_text(&self, expr: &expr::Expr) -> ast::Expr {
+        let ast_expr = self.expr(expr);
+        ast::Expr::Cast {
+            expr: Box::new(ast_expr),
+            data_type: ast::DataType::String(None),
+            format: None,
+        }
+    }
+    fn substr(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
+        assert!(exprs.len() == 2);
+        let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+        function_builder("SUBSTR", ast_exprs, false)
+    }
+    fn substr_with_size(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
+        assert!(exprs.len() == 3);
+        let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+        function_builder("SUBSTR", ast_exprs, false)
+    }
+    /// Converting MD5(X) to TO_HEX(MD5(X))
+    fn md5(&self, expr: &expr::Expr) -> ast::Expr {
+        let ast_expr = self.expr(expr);
+        let md5_function = function_builder("MD5", vec![ast_expr], false);
+        function_builder("TO_HEX", vec![md5_function], false)
+    }
+    fn random(&self) -> ast::Expr {
+        function_builder("RAND", vec![], false)
+    }
+    fn join_projection(&self, join: &Join) -> Vec<ast::SelectItem> {
+        join.left()
+            .schema()
+            .iter()
+            .map(|f| self.expr(&expr::Expr::qcol(Join::left_name(), f.name())))
+            .chain(
+                join.right()
+                    .schema()
+                    .iter()
+                    .map(|f| self.expr(&expr::Expr::qcol(Join::right_name(), f.name()))),
+            )
+            .zip(join.schema().iter())
+            .map(|(expr, field)| ast::SelectItem::ExprWithAlias {
+                expr,
+                alias: field.name().into(),
+            })
+            .collect()
+    }
+}
+
+impl QueryToRelationTranslator for BigQueryTranslator {
+    type D = BigQueryDialect;
+
+    fn dialect(&self) -> Self::D {
+        BigQueryDialect {}
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+    use crate::{
+        builder::{Ready, With},
+        data_type::{DataType, Value as _},
+        dialect_translation::RelationWithTranslator,
+        expr::Expr,
+        namer,
+        relation::{schema::Schema, Relation, Variant as _},
+    };
+    use std::sync::Arc;
+
+    fn assert_same_query_str(query_1: &str, query_2: &str) {
+        let a_no_whitespace: String = query_1.chars().filter(|c| !c.is_whitespace()).collect();
+        let b_no_whitespace: String = query_2.chars().filter(|c| !c.is_whitespace()).collect();
+        assert_eq!(a_no_whitespace, b_no_whitespace);
+    }
+
+    #[test]
+    fn test_rel_to_query() {
+        namer::reset();
+        let schema: Schema = vec![
+            ("a", DataType::float()),
+            ("b", DataType::float_interval(-2., 2.)),
+            ("c", DataType::float()),
+            ("d", DataType::float_interval(0., 1.)),
+        ]
+        .into_iter()
+        .collect();
+        let table: Arc<Relation> = Arc::new(
+            Relation::table()
+                .name("table")
+                .schema(schema.clone())
+                .size(100)
+                .build(),
+        );
+        let map: Arc<Relation> = Arc::new(
+            Relation::map()
+                .name("map_1")
+                .with(Expr::col("a"))
+                .input(table.clone())
+                .build(),
+        );
+        let rel_with_traslator = RelationWithTranslator(map.as_ref(), BigQueryTranslator);
+        let query = ast::Query::from(rel_with_traslator);
+        let translated = r#"
+            WITH map_1 AS (SELECT a AS field_s7n2 FROM table) SELECT * FROM map_1
+        "#;
+        assert_same_query_str(&query.to_string(), translated);
+    }
+}

src/dialect_translation/mod.rs

@@ -20,7 +20,7 @@ use crate::{
     data_type::DataTyped,
     expr::Identifier,
     hierarchy::Hierarchy,
-    relation::{JoinOperator, Table, Variant},
+    relation::{Join, JoinOperator, Table, Variant},
     sql::{self, parse, parse_with_dialect, Error, Result},
     DataType, Relation,
 };
@@ -266,6 +266,11 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
                     from: None,
                 }
             }
+            fn join_projection(&self, join: &Join) -> Vec<ast::SelectItem> {
+                vec![ast::SelectItem::Wildcard(
+                    ast::WildcardAdditionalOptions::default(),
+                )]
+            }
 
             fn identifier(&self, value: &expr::Identifier) -> Vec<ast::Ident> {
                 value.iter().map(ast::Ident::new).collect()
@@ -642,25 +647,28 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
             }
             fn position(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
                 assert!(exprs.len() == 2);
-                let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+                let ast_exprs: Vec<ast::Expr> =
+                    exprs.into_iter().map(|expr| self.expr(expr)).collect();
                 ast::Expr::Position {
                     expr: Box::new(ast_exprs[0].clone()),
                     r#in: Box::new(ast_exprs[1].clone()),
                 }
             }
             fn substr(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
-                assert!(exprs.len() == 3);
-                let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+                assert!(exprs.len() == 2);
+                let ast_exprs: Vec<ast::Expr> =
+                    exprs.into_iter().map(|expr| self.expr(expr)).collect();
                 ast::Expr::Substring {
                     expr: Box::new(ast_exprs[0].clone()),
                     substring_from: Some(Box::new(ast_exprs[1].clone())),
-                    substring_for: Some(Box::new(ast_exprs[2].clone())),
+                    substring_for: None,
                     special: false,
                 }
             }
             fn substr_with_size(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
                 assert!(exprs.len() == 3);
-                let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+                let ast_exprs: Vec<ast::Expr> =
+                    exprs.into_iter().map(|expr| self.expr(expr)).collect();
                 ast::Expr::Substring {
                     expr: Box::new(ast_exprs[0].clone()),
                     substring_from: Some(Box::new(ast_exprs[1].clone())),
@@ -674,7 +682,8 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
             }
             fn ilike(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
                 assert!(exprs.len() == 2);
-                let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+                let ast_exprs: Vec<ast::Expr> =
+                    exprs.into_iter().map(|expr| self.expr(expr)).collect();
                 ast::Expr::ILike {
                     negated: false,
                     expr: Box::new(ast_exprs[0].clone()),
@@ -684,7 +693,8 @@ macro_rules! relation_to_query_tranlator_trait_constructor {
             }
             fn like(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {
                 assert!(exprs.len() == 2);
-                let ast_exprs: Vec<ast::Expr> = exprs.into_iter().map(|expr| self.expr(expr)).collect();
+                let ast_exprs: Vec<ast::Expr> =
+                    exprs.into_iter().map(|expr| self.expr(expr)).collect();
                 ast::Expr::Like {
                     negated: false,
                     expr: Box::new(ast_exprs[0].clone()),

src/dialect_translation/mssql.rs

@@ -126,12 +126,12 @@ impl RelationToQueryTranslator for MsSqlTranslator {
         ast::Expr::from(&casted_to_integer)
     }
 
-    fn cast_as_text(&self,expr: &expr::Expr) -> ast::Expr {
+    fn cast_as_text(&self, expr: &expr::Expr) -> ast::Expr {
         let ast_expr = self.expr(expr);
         ast::Expr::Cast {
             expr: Box::new(ast_expr),
             data_type: ast::DataType::Nvarchar(Some(255)),
-            format: None
+            format: None,
         }
     }
     fn substr(&self, exprs: Vec<&expr::Expr>) -> ast::Expr {