Optimize filter executor in pull-based executor (#4421)

* optimzie filter executor

* Update datafusion/core/src/physical_plan/filter.rs

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>

datafusion/core/src/physical_plan/filter.rs

@@ -238,15 +238,32 @@ impl Stream for FilterExecStream {
         mut self: Pin<&mut Self>,
         cx: &mut Context<'_>,
     ) -> Poll<Option<Self::Item>> {
-        let poll = self.input.poll_next_unpin(cx).map(|x| match x {
-            Some(Ok(batch)) => {
-                let timer = self.baseline_metrics.elapsed_compute().timer();
-                let filtered_batch = batch_filter(&batch, &self.predicate);
-                timer.done();
-                Some(filtered_batch)
+        let poll;
+        loop {
+            match self.input.poll_next_unpin(cx) {
+                Poll::Ready(value) => match value {
+                    Some(Ok(batch)) => {
+                        let timer = self.baseline_metrics.elapsed_compute().timer();
+                        let filtered_batch = batch_filter(&batch, &self.predicate)?;
+                        // skip entirely filtered batches
+                        if filtered_batch.num_rows() == 0 {
+                            continue;
+                        }
+                        timer.done();
+                        poll = Poll::Ready(Some(Ok(filtered_batch)));
+                        break;
+                    }
+                    _ => {
+                        poll = Poll::Ready(value);
+                        break;
+                    }
+                },
+                Poll::Pending => {
+                    poll = Poll::Pending;
+                    break;
+                }
             }
-            other => other,
-        });
+        }
         self.baseline_metrics.record_poll(poll)
     }