Add non_scalar_dictionary_expresions feature (apache#4386)

tustvold · Jan 20, 2023 · b880421 · b880421
1 parent 6dce728
commit b880421
Show file tree

Hide file tree

Showing 6 changed files with 11 additions and 6 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -95,11 +95,11 @@ jobs:
       - name: Build tests
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --features avro,jit,scheduler,json --no-run
+          cargo test --features avro,jit,scheduler,json,non_scalar_dictionary_expressions --no-run
       - name: Run tests
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin
-          cargo test --features avro,jit,scheduler,json
+          cargo test --features avro,jit,scheduler,json,non_scalar_dictionary_expressions
       - name: Run examples
         run: |
           export PATH=$PATH:$HOME/d/protoc/bin

diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
@@ -53,6 +53,8 @@ regex_expressions = ["datafusion-physical-expr/regex_expressions"]
 scheduler = ["rayon"]
 simd = ["arrow/simd"]
 unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion-sql/unicode_expressions"]
+# Enables support for binary operations on dictionaries (note this results in significant additional codegen)
+non_scalar_dictionary_expressions = ["datafusion-physical-expr/non_scalar_dictionary_expressions"]
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
@@ -102,7 +104,6 @@ xz2 = { version = "0.1", optional = true }
 
 
 [dev-dependencies]
-arrow = { version = "31.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
 async-trait = "0.1.53"
 criterion = "0.4"
 csv = "1.1.6"

diff --git a/datafusion/core/tests/path_partition.rs b/datafusion/core/tests/path_partition.rs
@@ -204,7 +204,7 @@ async fn csv_filter_with_file_col() -> Result<()> {
     );
 
     let result = ctx
-        .sql("SELECT c1, c2 FROM t WHERE date='2021-10-27' and date!=c1 LIMIT 5")
+        .sql("SELECT c1, c2 FROM t WHERE date='2021-10-27' and c1!='2021-10-27' LIMIT 5")
         .await?
         .collect()
         .await?;

diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs
@@ -621,6 +621,7 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> {
 }
 
 #[tokio::test]
+#[cfg(feature = "non_scalar_dictionary_expressions")]
 async fn query_on_string_dictionary() -> Result<()> {
     // Test to ensure DataFusion can operate on dictionary types
     // Use StringDictionary (32 bit indexes = keys)

diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
@@ -24,7 +24,7 @@ repository = "https://github.com/apache/arrow-datafusion"
 readme = "README.md"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
 license = "Apache-2.0"
-keywords = [ "arrow", "query", "sql" ]
+keywords = ["arrow", "query", "sql"]
 edition = "2021"
 rust-version = "1.62"
 
@@ -37,10 +37,12 @@ crypto_expressions = ["md-5", "sha2", "blake2", "blake3"]
 default = ["crypto_expressions", "regex_expressions", "unicode_expressions"]
 regex_expressions = ["regex"]
 unicode_expressions = ["unicode-segmentation"]
+# Enables support for binary operations on dictionaries (note this results in significant additional codegen)
+non_scalar_dictionary_expressions = ["arrow/dyn_cmp_dict", "arrow/dyn_arith_dict"]
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "31.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { version = "31.0.0", features = ["prettyprint"] }
 arrow-buffer = "31.0.0"
 arrow-schema = "31.0.0"
 blake2 = { version = "^0.10.2", optional = true }

diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
@@ -1502,6 +1502,7 @@ mod tests {
     // is no way at the time of this writing to create a dictionary
     // array using the `From` trait
     #[test]
+    #[cfg(feature = "non_scalar_dictionary_expressions")]
     fn test_dictionary_type_to_array_coersion() -> Result<()> {
         // Test string  a string dictionary
         let dict_type =