perf: reduce amount of stack memory usage (#4103)

prqlc/prqlc/src/semantic/resolver/expr.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use itertools::Itertools;
 
-use prqlc_ast::{TupleField, Ty, TyKind};
+use prqlc_ast::{Span, TupleField, Ty, TyKind};
 
 use crate::ir::decl::{DeclKind, Module};
 use crate::ir::pl::*;
@@ -63,10 +63,10 @@ impl PlFold for Resolver<'_> {
         }
 
         let id = self.id.gen();
-        let alias = node.alias.clone();
-        let span = node.span;
+        let alias = Box::new(node.alias.clone());
+        let span = Box::new(node.span);
 
-        if let Some(span) = span {
+        if let Some(span) = *span {
             self.root_mod.span_map.insert(id, span);
         }
 
@@ -108,9 +108,9 @@ impl PlFold for Resolver<'_> {
 
                     DeclKind::Expr(expr) => match &expr.kind {
                         ExprKind::Func(closure) => {
-                            let closure = self.fold_function_types(*closure.clone())?;
+                            let closure = self.fold_function_types(closure.clone())?;
 
-                            let expr = Expr::new(ExprKind::Func(Box::new(closure)));
+                            let expr = Expr::new(ExprKind::Func(closure));
 
                             if self.in_func_call_name {
                                 expr
@@ -139,7 +139,7 @@ impl PlFold for Resolver<'_> {
                             expected: "a value".to_string(),
                             found: "a type".to_string(),
                         })
-                        .with_span(span)
+                        .with_span(*span)
                         .into());
                     }
 
@@ -167,17 +167,17 @@ impl PlFold for Resolver<'_> {
                 self.default_namespace = None;
                 let old = self.in_func_call_name;
                 self.in_func_call_name = true;
-                let name = self.fold_expr(*name)?;
+                let name = Box::new(self.fold_expr(*name)?);
                 self.in_func_call_name = old;
 
-                let func = *name.try_cast(|n| n.into_func(), None, "a function")?;
+                let func = name.try_cast(|n| n.into_func(), None, "a function")?;
 
                 // fold function
                 let func = self.apply_args_to_closure(func, args, named_args)?;
-                self.fold_function(func, span)?
+                self.fold_function(func, *span)?
             }
 
-            ExprKind::Func(closure) => self.fold_function(*closure, span)?,
+            ExprKind::Func(closure) => self.fold_function(closure, *span)?,
 
             ExprKind::Tuple(exprs) => {
                 let exprs = self.fold_exprs(exprs)?;
@@ -202,7 +202,20 @@ impl PlFold for Resolver<'_> {
                 ..node
             },
         };
-        let mut r = self.static_eval(r)?;
+        self.finish_expr_resolve(r, id, *alias, *span)
+    }
+}
+
+impl Resolver<'_> {
+    fn finish_expr_resolve(
+        &mut self,
+        expr: Expr,
+        id: usize,
+        alias: Option<String>,
+        span: Option<Span>,
+    ) -> Result<Expr> {
+        let mut r = Box::new(self.static_eval(expr)?);
+
         r.id = r.id.or(Some(id));
         r.alias = r.alias.or(alias);
         r.span = r.span.or(span);
@@ -233,11 +246,9 @@ impl PlFold for Resolver<'_> {
                 }
             }
         }
-        Ok(r)
+        Ok(*r)
     }
-}
 
-impl Resolver<'_> {
     pub fn resolve_column_exclusion(&mut self, expr: Expr) -> Result<Expr> {
         let expr = self.fold_expr(expr)?;
         let except = self.coerce_into_tuple(expr)?;

prqlc/prqlc/src/semantic/resolver/functions.rs

@@ -15,7 +15,7 @@ use crate::{Error, Span, WithErrorInfo};
 use super::Resolver;
 
 impl Resolver<'_> {
-    pub fn fold_function(&mut self, closure: Func, span: Option<Span>) -> Result<Expr> {
+    pub fn fold_function(&mut self, closure: Box<Func>, span: Option<Span>) -> Result<Expr> {
         let closure = self.fold_function_types(closure)?;
 
         log::debug!(
@@ -36,7 +36,7 @@ impl Resolver<'_> {
 
         let enough_args = closure.args.len() == closure.params.len();
         if !enough_args {
-            return Ok(expr_of_func(closure, span));
+            return Ok(*expr_of_func(closure, span));
         }
 
         // make sure named args are pushed into params
@@ -49,10 +49,10 @@ impl Resolver<'_> {
         // push the env
         let closure_env = Module::from_exprs(closure.env);
         self.root_mod.module.stack_push(NS_PARAM, closure_env);
-        let closure = Func {
+        let closure = Box::new(Func {
             env: HashMap::new(),
-            ..closure
-        };
+            ..*closure
+        });
 
         if log::log_enabled!(log::Level::Debug) {
             let name = closure
@@ -66,7 +66,7 @@ impl Resolver<'_> {
         let closure = match res {
             Ok(func) => func,
             Err(func) => {
-                return Ok(expr_of_func(func, span));
+                return Ok(*expr_of_func(func, span));
             }
         };
 
@@ -94,50 +94,55 @@ impl Resolver<'_> {
             }
         } else {
             // base case: materialize
-            log::debug!("stack_push for {}", closure.as_debug_name());
+            self.materialize_function(closure)?
+        };
 
-            let (func_env, body) = env_of_closure(closure);
+        // pop the env
+        self.root_mod.module.stack_pop(NS_PARAM).unwrap();
 
-            self.root_mod.module.stack_push(NS_PARAM, func_env);
+        Ok(Expr { span, ..res })
+    }
 
-            // fold again, to resolve inner variables & functions
-            let body = self.fold_expr(body)?;
+    #[allow(clippy::boxed_local)]
+    fn materialize_function(&mut self, closure: Box<Func>) -> Result<Expr> {
+        log::debug!("stack_push for {}", closure.as_debug_name());
 
-            // remove param decls
-            log::debug!("stack_pop: {:?}", body.id);
-            let func_env = self.root_mod.module.stack_pop(NS_PARAM).unwrap();
+        let (func_env, body) = env_of_closure(*closure);
 
-            if let ExprKind::Func(mut inner_closure) = body.kind {
-                // body couldn't been resolved - construct a closure to be evaluated later
+        self.root_mod.module.stack_push(NS_PARAM, func_env);
 
-                inner_closure.env = func_env.into_exprs();
+        // fold again, to resolve inner variables & functions
+        let body = self.fold_expr(body)?;
 
-                let (got, missing) = inner_closure.params.split_at(inner_closure.args.len());
-                let missing = missing.to_vec();
-                inner_closure.params = got.to_vec();
+        // remove param decls
+        log::debug!("stack_pop: {:?}", body.id);
+        let func_env = self.root_mod.module.stack_pop(NS_PARAM).unwrap();
 
-                Expr::new(ExprKind::Func(Box::new(Func {
-                    name_hint: None,
-                    args: vec![],
-                    params: missing,
-                    named_params: vec![],
-                    body: Box::new(Expr::new(ExprKind::Func(inner_closure))),
-                    return_ty: None,
-                    env: HashMap::new(),
-                })))
-            } else {
-                // resolved, return result
-                body
-            }
-        };
+        Ok(if let ExprKind::Func(mut inner_closure) = body.kind {
+            // body couldn't been resolved - construct a closure to be evaluated later
 
-        // pop the env
-        self.root_mod.module.stack_pop(NS_PARAM).unwrap();
+            inner_closure.env = func_env.into_exprs();
 
-        Ok(Expr { span, ..res })
+            let (got, missing) = inner_closure.params.split_at(inner_closure.args.len());
+            let missing = missing.to_vec();
+            inner_closure.params = got.to_vec();
+
+            Expr::new(ExprKind::Func(Box::new(Func {
+                name_hint: None,
+                args: vec![],
+                params: missing,
+                named_params: vec![],
+                body: Box::new(Expr::new(ExprKind::Func(inner_closure))),
+                return_ty: None,
+                env: HashMap::new(),
+            })))
+        } else {
+            // resolved, return result
+            body
+        })
     }
 
-    pub fn fold_function_types(&mut self, mut closure: Func) -> Result<Func> {
+    pub fn fold_function_types(&mut self, mut closure: Box<Func>) -> Result<Box<Func>> {
         closure.params = closure
             .params
             .into_iter()
@@ -154,10 +159,10 @@ impl Resolver<'_> {
 
     pub fn apply_args_to_closure(
         &mut self,
-        mut closure: Func,
+        mut closure: Box<Func>,
         args: Vec<Expr>,
         mut named_args: HashMap<String, Expr>,
-    ) -> Result<Func> {
+    ) -> Result<Box<Func>> {
         // named arguments are consumed only by the first function
 
         // named
@@ -184,11 +189,14 @@ impl Resolver<'_> {
     }
 
     /// Resolves function arguments. Will return `Err(func)` is partial application is required.
-    fn resolve_function_args(&mut self, to_resolve: Func) -> Result<Result<Func, Func>> {
-        let mut closure = Func {
+    fn resolve_function_args(
+        &mut self,
+        #[allow(clippy::boxed_local)] to_resolve: Box<Func>,
+    ) -> Result<Result<Box<Func>, Box<Func>>> {
+        let mut closure = Box::new(Func {
             args: vec![Expr::new(Literal::Null); to_resolve.args.len()],
-            ..to_resolve
-        };
+            ..*to_resolve
+        });
         let mut partial_application_position = None;
 
         let func_name = &closure.name_hint;
@@ -342,7 +350,7 @@ impl Resolver<'_> {
     }
 }
 
-fn extract_partial_application(mut func: Func, position: usize) -> Func {
+fn extract_partial_application(mut func: Box<Func>, position: usize) -> Box<Func> {
     // Input:
     // Func {
     //     params: [x, y, z],
@@ -393,10 +401,10 @@ fn extract_partial_application(mut func: Func, position: usize) -> Func {
     arg_func.args.push(substitute_arg);
 
     // set the arg func body to the parent func
-    Func {
+    Box::new(Func {
         name_hint: None,
         return_ty: None,
-        body: Box::new(Expr::new(func)),
+        body: Box::new(Expr::new(ExprKind::Func(func))),
         params: vec![FuncParam {
             name: param_name,
             ty: None,
@@ -405,7 +413,7 @@ fn extract_partial_application(mut func: Func, position: usize) -> Func {
         named_params: Default::default(),
         args: Default::default(),
         env: Default::default(),
-    }
+    })
 }
 
 fn env_of_closure(closure: Func) -> (Module, Expr) {
@@ -424,7 +432,7 @@ fn env_of_closure(closure: Func) -> (Module, Expr) {
     (func_env, *closure.body)
 }
 
-pub fn expr_of_func(func: Func, span: Option<Span>) -> Expr {
+pub fn expr_of_func(func: Box<Func>, span: Option<Span>) -> Box<Expr> {
     let ty = TyFunc {
         args: func
             .params
@@ -436,9 +444,9 @@ pub fn expr_of_func(func: Func, span: Option<Span>) -> Expr {
         name_hint: func.name_hint.clone(),
     };
 
-    Expr {
+    Box::new(Expr {
         ty: Some(Ty::new(ty)),
         span,
-        ..Expr::new(ExprKind::Func(Box::new(func)))
-    }
+        ..Expr::new(ExprKind::Func(func))
+    })
 }

prqlc/prqlc/src/semantic/resolver/transforms.rs

@@ -23,7 +23,8 @@ use super::Resolver;
 
 impl Resolver<'_> {
     /// try to convert function call with enough args into transform
-    pub fn resolve_special_func(&mut self, func: Func, needs_window: bool) -> Result<Expr> {
+    #[allow(clippy::boxed_local)]
+    pub fn resolve_special_func(&mut self, func: Box<Func>, needs_window: bool) -> Result<Expr> {
         let internal_name = func.body.kind.into_internal().unwrap();
 
         let (kind, input) = match internal_name.as_str() {
@@ -623,7 +624,7 @@ impl Resolver<'_> {
         })?;
 
         // construct the function back
-        let func = Func {
+        let func = Box::new(Func {
             name_hint: None,
             body: Box::new(pipeline),
             return_ty: None,
@@ -637,8 +638,8 @@ impl Resolver<'_> {
             named_params: vec![],
 
             env: Default::default(),
-        };
-        Ok(expr_of_func(func, span))
+        });
+        Ok(*expr_of_func(func, span))
     }
 }
 

prqlc/prqlc/tests/integration/cli.rs

@@ -128,7 +128,6 @@ fn compile_help() {
     "###);
 }
 
-#[cfg(not(windows))] // This is back to causing a SO on Windows since https://github.com/PRQL/prql/pull/3786
 #[test]
 fn long_query() {
     assert_cmd_snapshot!(prqlc_command()