Fix large futures causing stack overflows (#10033)

* Enable clippy::pedantic::large_futures lint

* Lighten DataFrame size and fix large futures warnings

Cargo.toml

@@ -126,3 +126,7 @@ opt-level = 3
 overflow-checks = false
 panic = 'unwind'
 rpath = false
+
+[workspace.lints.clippy]
+# Detects large stack-allocated futures that may cause stack overflow crashes (see threshold in clippy.toml)
+large_futures = "warn"

benchmarks/Cargo.toml

@@ -26,6 +26,9 @@ repository = { workspace = true }
 license = { workspace = true }
 rust-version = { workspace = true }
 
+[lints]
+workspace = true
+
 [features]
 ci = []
 default = ["mimalloc"]

clippy.toml

@@ -6,3 +6,7 @@ disallowed-methods = [
 disallowed-types = [
     { path = "std::time::Instant", reason = "Use `datafusion_common::instant::Instant` instead for WASM compatibility" },
 ]
+
+# Lowering the threshold to help prevent stack overflows (default is 16384)
+# See: https://rust-lang.github.io/rust-clippy/master/index.html#/large_futures
+future-size-threshold = 10000

datafusion-examples/Cargo.toml

@@ -29,6 +29,9 @@ license = { workspace = true }
 authors = { workspace = true }
 rust-version = { workspace = true }
 
+[lints]
+workspace = true
+
 [[example]]
 name = "flight_sql_server"
 path = "examples/flight/flight_sql_server.rs"

datafusion/common-runtime/Cargo.toml

@@ -28,6 +28,9 @@ license = { workspace = true }
 authors = { workspace = true }
 rust-version = { workspace = true }
 
+[lints]
+workspace = true
+
 [lib]
 name = "datafusion_common_runtime"
 path = "src/lib.rs"

datafusion/common/Cargo.toml

@@ -28,6 +28,9 @@ license = { workspace = true }
 authors = { workspace = true }
 rust-version = { workspace = true }
 
+[lints]
+workspace = true
+
 [lib]
 name = "datafusion_common"
 path = "src/lib.rs"

datafusion/core/Cargo.toml

@@ -32,6 +32,9 @@ authors = { workspace = true }
 # https://github.com/foresterre/cargo-msrv/issues/590
 rust-version = "1.73"
 
+[lints]
+workspace = true
+
 [lib]
 name = "datafusion"
 path = "src/lib.rs"