From 3288dc52ec578d6bbc0e20423f2a4165bfbb499a Mon Sep 17 00:00:00 2001 From: wlandau-lilly Date: Wed, 8 Aug 2018 12:09:54 -0400 Subject: [PATCH] Expose the template argument of clustermq functions ref: https://github.com/ropensci/drake/issues/425#issuecomment-402363782 cc @kendonB --- R/config.R | 17 +++++++++++++++-- R/make.R | 6 ++++-- R/staged.R | 5 ++++- man/drake_config.Rd | 14 +++++++++++++- man/make.Rd | 14 +++++++++++++- 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/R/config.R b/R/config.R index 79894d9e0..9428c15df 100644 --- a/R/config.R +++ b/R/config.R @@ -360,6 +360,17 @@ #' @param garbage_collection logical, whether to call `gc()` each time #' a target is built during [make()]. #' +#' @param template a named list of values to fill in the `{{ ... }}` +#' placeholders in template files (e.g. from [drake_hpc_template_file()]). +#' Same as the `template` argument of `clustermq::Q()` and +#' `clustermq::workers`. +#' Enabled for `clustermq` only (`make(parallelism = "clustermq_staged")`), +#' not `future` or `batchtools` so far. +#' For more information, see the `clustermq` package: +#' . +#' Some template placeholders such as `{{ job_name }}` and `{{ n_jobs }}` +#' cannot be set this way. +#' #' @examples #' \dontrun{ #' test_with_dir("Quarantine side effects.", { @@ -417,7 +428,8 @@ drake_config <- function( makefile_path = "Makefile", console_log_file = NULL, ensure_workers = TRUE, - garbage_collection = FALSE + garbage_collection = FALSE, + template = list() ){ force(envir) unlink(console_log_file) @@ -522,7 +534,8 @@ drake_config <- function( ensure_workers = ensure_workers, all_targets = all_targets, all_imports = all_imports, - garbage_collection = garbage_collection + garbage_collection = garbage_collection, + template = template ) } diff --git a/R/make.R b/R/make.R index 3b9984e7e..0b24f6ab4 100644 --- a/R/make.R +++ b/R/make.R @@ -114,7 +114,8 @@ make <- function( makefile_path = "Makefile", console_log_file = NULL, ensure_workers = TRUE, - garbage_collection = FALSE + garbage_collection = FALSE, + template = list() ){ force(envir) if (!is.null(return_config)){ @@ -164,7 +165,8 @@ make <- function( makefile_path = makefile_path, console_log_file = console_log_file, ensure_workers = ensure_workers, - garbage_collection = garbage_collection + garbage_collection = garbage_collection, + template = template ) } make_with_config(config = config) diff --git a/R/staged.R b/R/staged.R index 518dcc518..bbeeecbba 100644 --- a/R/staged.R +++ b/R/staged.R @@ -167,7 +167,10 @@ run_clustermq_staged <- function(config){ assert_pkgs("clustermq") schedule <- config$schedule withr::with_preserve_seed( - workers <- clustermq::workers(n_jobs = config$jobs) + workers <- clustermq::workers( + n_jobs = config$jobs, + template = config$template + ) ) on.exit(workers$finalize()) while (length(V(schedule)$name)){ diff --git a/man/drake_config.Rd b/man/drake_config.Rd index 40f8251c3..aa19d617d 100644 --- a/man/drake_config.Rd +++ b/man/drake_config.Rd @@ -23,7 +23,8 @@ drake_config(plan = read_drake_plan(), targets = NULL, session = NULL, imports_only = NULL, pruning_strategy = c("lookahead", "speed", "memory"), makefile_path = "Makefile", console_log_file = NULL, - ensure_workers = TRUE, garbage_collection = FALSE) + ensure_workers = TRUE, garbage_collection = FALSE, + template = list()) } \arguments{ \item{plan}{workflow plan data frame. @@ -374,6 +375,17 @@ This argument only applies to parallel computing with persistent workers \item{garbage_collection}{logical, whether to call \code{gc()} each time a target is built during \code{\link[=make]{make()}}.} + +\item{template}{a named list of values to fill in the \code{{{ ... }}} +placeholders in template files (e.g. from \code{\link[=drake_hpc_template_file]{drake_hpc_template_file()}}). +Same as the \code{template} argument of \code{clustermq::Q()} and +\code{clustermq::workers}. +Enabled for \code{clustermq} only (\code{make(parallelism = "clustermq_staged")}), +not \code{future} or \code{batchtools} so far. +For more information, see the \code{clustermq} package: +\url{https://github.com/mschubert/clustermq}. +Some template placeholders such as \code{{{ job_name }}} and \code{{{ n_jobs }}} +cannot be set this way.} } \value{ The master internal configuration list of a project. diff --git a/man/make.Rd b/man/make.Rd index 6833b9667..f0966e323 100644 --- a/man/make.Rd +++ b/man/make.Rd @@ -22,7 +22,8 @@ make(plan = read_drake_plan(), targets = NULL, session = NULL, imports_only = NULL, pruning_strategy = c("lookahead", "speed", "memory"), makefile_path = "Makefile", console_log_file = NULL, - ensure_workers = TRUE, garbage_collection = FALSE) + ensure_workers = TRUE, garbage_collection = FALSE, + template = list()) } \arguments{ \item{plan}{workflow plan data frame. @@ -381,6 +382,17 @@ This argument only applies to parallel computing with persistent workers \item{garbage_collection}{logical, whether to call \code{gc()} each time a target is built during \code{\link[=make]{make()}}.} + +\item{template}{a named list of values to fill in the \code{{{ ... }}} +placeholders in template files (e.g. from \code{\link[=drake_hpc_template_file]{drake_hpc_template_file()}}). +Same as the \code{template} argument of \code{clustermq::Q()} and +\code{clustermq::workers}. +Enabled for \code{clustermq} only (\code{make(parallelism = "clustermq_staged")}), +not \code{future} or \code{batchtools} so far. +For more information, see the \code{clustermq} package: +\url{https://github.com/mschubert/clustermq}. +Some template placeholders such as \code{{{ job_name }}} and \code{{{ n_jobs }}} +cannot be set this way.} } \value{ The master internal configuration list, mostly