Remove reundant filters (e.g. c> 5 AND c>5 --> c>5) (#436)

* #410: Remove reundant filters (e.g. c> 5 AND c>5 --> c>5) * RemoveDuplicateFilters: fix unit tests * fix lint * fix erroneous simplifications of arithmetic expressions * RemoveDuplicateFilters: add more simplification rules (@Dandandan) * Remove unnecessary lifetime specifiers and other fixes ... * RemoveDuplicateFilter: fix is_one matches * Change remove_duplicate_filter to simplify_expressions * fix simplify expressions * fix lint and clippy * Update datafusion/src/optimizer/simplify_expressions.rs Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org> * Update datafusion/src/optimizer/simplify_expressions.rs Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org> * Update datafusion/src/optimizer/simplify_expressions.rs Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org> * Update datafusion/src/optimizer/simplify_expressions.rs Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org> * simplify test expressions Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
apache · Jun 3, 2021 · 4392eea · 4392eea
1 parent a1b8305
commit 4392eea
Show file tree

Hide file tree

Showing 3 changed files with 544 additions and 0 deletions.
diff --git a/datafusion/src/execution/context.rs b/datafusion/src/execution/context.rs
@@ -59,6 +59,7 @@ use crate::optimizer::filter_push_down::FilterPushDown;
 use crate::optimizer::limit_push_down::LimitPushDown;
 use crate::optimizer::optimizer::OptimizerRule;
 use crate::optimizer::projection_push_down::ProjectionPushDown;
+use crate::optimizer::simplify_expressions::SimplifyExpressions;
 use crate::physical_optimizer::coalesce_batches::CoalesceBatches;
 use crate::physical_optimizer::merge_exec::AddMergeExec;
 use crate::physical_optimizer::repartition::Repartition;
@@ -652,6 +653,7 @@ impl ExecutionConfig {
                 Arc::new(EliminateLimit::new()),
                 Arc::new(ProjectionPushDown::new()),
                 Arc::new(FilterPushDown::new()),
+                Arc::new(SimplifyExpressions::new()),
                 Arc::new(HashBuildProbeOrder::new()),
                 Arc::new(LimitPushDown::new()),
             ],

diff --git a/datafusion/src/optimizer/mod.rs b/datafusion/src/optimizer/mod.rs
@@ -25,4 +25,5 @@ pub mod hash_build_probe_order;
 pub mod limit_push_down;
 pub mod optimizer;
 pub mod projection_push_down;
+pub mod simplify_expressions;
 pub mod utils;