Merge pull request #395 from ropensci/i385

Add alternative memory management strategies

NEWS.md

@@ -26,6 +26,7 @@
 - Use a `README.Rmd` file to generate `README.md`.
 - Add function `deps_targets()`.
 - Deprecate function `deps()` in favor of `deps_code()`
+- Add a `pruning_strategy` argument to `make()` and `drake_config()` so the user can decide how `drake` keeps non-import dependencies in memory when it builds a target.
 
 # Version 5.1.2
 

R/config.R

@@ -322,6 +322,19 @@
 #'   but it causes [make()] to populate your workspace/environment
 #'   with the last few targets it builds.
 #'
+#' @param pruning_strategy Character scalar, either `"speed"` (default)
+#'   or `"memory"`. These are alternative approaches to how `drake`
+#'   keeps non-import dependencies in memory when it builds a target.
+#'   If `pruning_strategy` is `"memory"`, `drake` removes all targets
+#'   from memory (i.e. `config$envir`) except the direct dependencies
+#'   of the target is about to build. This is suitable for data so large
+#'   that the optimal strategy is to minimize memory consumption.
+#'   If `pruning_strategy` is `"speed"`, `drake` loads all the dependencies
+#'   and keeps in memory everything that will eventually be a
+#'   dependency of a downstream target. This strategy consumes more
+#'   memory, but does more to minimize the number of times data is
+#'   read from storage/disk.
+#'
 #' @examples
 #' \dontrun{
 #' test_with_dir("Quarantine side effects.", {
@@ -377,7 +390,8 @@ drake_config <- function(
   caching = c("worker", "master"),
   keep_going = FALSE,
   session = NULL,
-  imports_only = NULL
+  imports_only = NULL,
+  pruning_strategy = c("speed", "memory")
 ){
   force(envir)
   if (!is.null(imports_only)){
@@ -418,6 +432,7 @@ drake_config <- function(
   }
   cache_path <- force_cache_path(cache)
   lazy_load <- parse_lazy_arg(lazy_load)
+  pruning_strategy <- match.arg(pruning_strategy)
   list(
     plan = plan, targets = targets, envir = envir,
     cache = cache, cache_path = cache_path, fetch_cache = fetch_cache,
@@ -433,7 +448,7 @@ drake_config <- function(
     lazy_load = lazy_load, session_info = session_info,
     cache_log_file = cache_log_file, caching = match.arg(caching),
     evaluator = future::plan("next"), keep_going = keep_going,
-    session = session
+    session = session, pruning_strategy = pruning_strategy
   )
 }
 

R/envir.R

@@ -26,12 +26,14 @@ assign_to_envir <- function(target, value, config){
 #' @examples
 #' # Users should use make().
 prune_envir <- function(targets, config, downstream = NULL, jobs = 1){
-  if (is.null(downstream)){
+  if (is.null(downstream) && config$pruning_strategy == "speed"){
     downstream <- downstream_nodes(
       from = targets,
       graph = config$graph,
       jobs = jobs
     )
+  } else if (config$pruning_strategy == "memory"){
+    downstream <- NULL
   }
   already_loaded <- ls(envir = config$envir, all.names = TRUE) %>%
     intersect(y = config$plan$target)

R/make.R

@@ -108,7 +108,8 @@ make <- function(
   caching = "worker",
   keep_going = FALSE,
   session = NULL,
-  imports_only = NULL
+  imports_only = NULL,
+  pruning_strategy = c("speed", "memory")
 ){
   force(envir)
   if (!is.null(return_config)){
@@ -153,7 +154,8 @@ make <- function(
       caching = caching,
       keep_going = keep_going,
       session = session,
-      imports_only = imports_only
+      imports_only = imports_only,
+      pruning_strategy = pruning_strategy
     )
   }
   make_with_config(config = config)

R/migrate.R

@@ -89,6 +89,7 @@ migrate_drake_project <- function(
   config$outdated <- legacy_outdated(config) %>%
     as.character %>%
     sort
+  config$pruning_strategy <- "speed"
   config$cache$clear(namespace = "depends")
   store_drake_config(config = config)
   run_loop(config = config)