Implement parquet page-level skipping with column index, using min/ma… (

#3780) * Implement parquet page-level skipping with column index, using min/max stats Signed-off-by: yangjiang <yangjiang@ebay.com> * add test Signed-off-by: yangjiang <yangjiang@ebay.com> * support some types Signed-off-by: yangjiang <yangjiang@ebay.com> * fix typo Signed-off-by: yangjiang <yangjiang@ebay.com> * add comments Signed-off-by: yangjiang <yangjiang@ebay.com> Signed-off-by: yangjiang <yangjiang@ebay.com>
apache · Oct 15, 2022 · 011bcf4 · 011bcf4
1 parent aa0ded7
commit 011bcf4
Show file tree

Hide file tree

Showing 2 changed files with 340 additions and 3 deletions.
diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs
@@ -223,6 +223,20 @@ impl PruningPredicate {
     pub fn predicate_expr(&self) -> &Arc<dyn PhysicalExpr> {
         &self.predicate_expr
     }
+
+    /// Returns all need column indexes to evaluate this pruning predicate
+    pub(crate) fn need_input_columns_ids(&self) -> HashSet<usize> {
+        let mut set = HashSet::new();
+        self.required_columns.columns.iter().for_each(|x| {
+            match self.schema().column_with_name(x.0.name.as_str()) {
+                None => {}
+                Some(y) => {
+                    set.insert(y.0);
+                }
+            }
+        });
+        set
+    }
 }
 
 /// Handles creating references to the min/max statistics