Cache requests (#132)

* add caching of requests

* add news item

* add note about caching/downloading

* Update documentation

* add memoise to imports

---------

Co-authored-by: sbfnk <sbfnk@users.noreply.github.com>

DESCRIPTION

@@ -28,6 +28,7 @@ Imports:
     httr,
     jsonlite,
     lubridate,
+    memoise,
     oai,
     wpp2017,
     xml2,
@@ -46,7 +47,7 @@ VignetteBuilder:
 Encoding: UTF-8
 LazyData: true
 NeedsCompilation: no
-RoxygenNote: 7.3.0
+RoxygenNote: 7.3.2
 URL: https://github.com/epiforecasts/socialmixr,
     https://epiforecasts.io/socialmixr/
 BugReports: https://github.com/epiforecasts/socialmixr/issues

NAMESPACE

@@ -47,6 +47,7 @@ importFrom(lifecycle,deprecate_warn)
 importFrom(lubridate,period)
 importFrom(lubridate,period_to_seconds)
 importFrom(lubridate,years)
+importFrom(memoise,memoise)
 importFrom(oai,list_records)
 importFrom(stats,median)
 importFrom(stats,runif)

NEWS.md

@@ -2,6 +2,7 @@
 
 * The speed of loading surveys has been increased.
 * An error has been fixed causing NA contact matrices if any 5-year age band in the population data was missing.
+* Results of function calls accessing Zenodo repository are now cached for speedup and to avoid multiple web requests
 
 # socialmixr 0.3.2
 

R/get_survey.r

@@ -1,17 +1,33 @@
 #' Get a survey, either from its Zenodo repository, a set of files, or a survey variable
+#
 #'
-#' @description Downloads survey data, or extracts them from files, and returns a clean data set.
-#' @param survey a DOI or url to get the survey from, or a [survey()] object (in which case only cleaning is done).
-#' @param ... options for [clean()], which is called at the end of this
-#' @autoglobal
+#' Downloads survey data, or extracts them from files, and returns a clean data set. If a survey URL is accessed multiple times, the data will be cached (unless `clear_cache` is set to `TRUE`) to avoid repeated downloads.
+#'
+#' If survey objects are used repeatedly the downloaded files can be saved and reloaded between sessions then survey objects can be saved/loaded using [base::saveRDS()] and [base::readRDS()], or via the individual survey files that can be downloaded using [download_survey()] and subsequently loaded using [load_survey()].
+#' @param clear_cache logical, whether to clear the cache before downloading the survey; by default, the cache is not cleared and so multiple calls of this function to access the same survey will not result in repeated downloads
+#' @importFrom memoise memoise
+#' @inheritParams .get_survey
 #' @examples
 #' \dontrun{
 #' list_surveys()
 #' peru_survey <- get_survey("https://doi.org/10.5281/zenodo.1095664")
 #' }
 #' @return a survey in the correct format
 #' @export
-get_survey <- function(survey, ...) {
+get_survey <- function(survey, clear_cache = FALSE, ...) {
+  if (!("get_survey" %in% names(.socialmixr.env$cached_functions)) ||
+      clear_cache) {
+    .socialmixr.env$cached_functions$get_survey <- memoise(.get_survey)
+  }
+  .socialmixr.env$cached_functions$get_survey(survey, ...)
+}
+
+#' Internal function to get survey data
+#' @autoglobal
+#' @param survey a DOI or url to get the survey from, or a [survey()] object (in which case only cleaning is done).
+#' @param ... options for [clean()], which is called at the end of this
+#' @keywords internal
+.get_survey <- function(survey, ...) {
   if (inherits(survey, "survey")) {
     new_survey <- survey
   } else {

R/globals.R

@@ -24,12 +24,12 @@ utils::globalVariables(c(
   "proportion", # <contact_matrix>
   "participants", # <contact_matrix>
   "file_name", # <download_survey>
-  "common_doi", # <list_surveys>
-  "doi.nb", # <list_surveys>
-  "identifier.1", # <list_surveys>
-  "title", # <list_surveys>
-  "creator", # <list_surveys>
-  "identifier.2", # <list_surveys>
+  "common_doi", # <.list_surveys>
+  "doi.nb", # <.list_surveys>
+  "identifier.1", # <.list_surveys>
+  "title", # <.list_surveys>
+  "creator", # <.list_surveys>
+  "identifier.2", # <.list_surveys>
   "..main_id", # <load_survey>
   "..merge_id", # <load_survey>
   "..original.lower.age.limit", # <pop_age>

R/lists.r

@@ -1,14 +1,24 @@
 #' List all surveys available for download
 #'
 #' @return character vector of surveys
-#' @importFrom oai list_records
-#' @autoglobal
+#' @inheritParams get_survey
 #' @examples
 #' \dontrun{
 #' list_surveys()
 #' }
 #' @export
-list_surveys <- function() {
+list_surveys <- function(clear_cache = FALSE) {
+  if (!("list_surveys" %in% names(.socialmixr.env$cached_functions)) ||
+      clear_cache) {
+    .socialmixr.env$cached_functions$list_surveys <- memoise(.list_surveys)
+  }
+  .socialmixr.env$cached_functions$list_surveys()
+}
+
+#' @autoglobal
+#' @importFrom oai list_records
+#' @keywords internal
+.list_surveys <- function() {
   record_list <-
     data.table(list_records("https://zenodo.org/oai2d",
       metadataPrefix = "oai_datacite",

R/socialmixr-package.R

@@ -0,0 +1,2 @@
+.socialmixr.env <- new.env(parent = emptyenv())
+assign("cached_functions", list(), envir = .socialmixr.env)