Add in test environment connections

NAMESPACE

@@ -34,6 +34,8 @@ export(parse_tourl_filter_in)
 export(post_dataset)
 export(preview_dataset)
 export(query_dataset)
+export(validate_api_version)
+export(validate_ees_environment)
 export(validate_ees_filter_type)
 export(validate_ees_id)
 export(validate_page_size)

R/api_url.R

@@ -23,7 +23,7 @@
 #' @param page_size Number of results to return in a single query
 #' @param page Page number of query results to return
 #' @param api_version EES API version
-#' @param environment EES environment to connect to: "dev", "test", "preprod" or "prod"
+#' @param ees_environment EES ees_environment to connect to: "dev", "test", "preprod" or "prod"
 #' @param verbose Run with additional contextual messaging. Logical, default = FALSE
 #' @return A string containing the URL for connecting to the EES API
 #' @export
@@ -58,47 +58,32 @@ api_url <- function(
     locations = NULL,
     filter_items = NULL,
     dataset_version = NULL,
+    ees_environment = NULL,
+    api_version = NULL,
     page_size = NULL,
     page = NULL,
-    api_version = "1.0",
-    environment = "dev",
     verbose = FALSE) {
-  # Check that the API version is valid
-  is_valid_api_version <- function(vapi) {
-    !grepl(
-      "[a-z_%+-]",
-      as.character(vapi),
-      ignore.case = TRUE
-    )
+  # Creating a master switch here for ees_environment, so that when we switch from dev to test and
+  # then subsequently from test to prod, we can just change it here and everything should follow
+  # from here. ees_environment should default to NULL for most other functions. Probably not a
+  # "proper" way to do this as it's not clear from the primary user-facing functions themselves
+  # what it's going to default to, so probably want to remove this and do something better once
+  # we're through development.
+  if (is.null(ees_environment)) {
+    ees_environment <- default_ees_environment()
   }
+  validate_ees_environment(ees_environment)
 
-  if (is_valid_api_version(api_version) == FALSE) {
-    stop(
-      "You have entered an invalid API version in the api_version argument.
-      This should be numerical values only."
-    )
+  # Creating a master switch here for api_version. The default for this param should be set to NULL
+  # for most other functions, but can be set to the latest version here. We'll want to automate
+  # this once we know how to find out the latest api version from the api itself.
+  if (is.null(api_version)) {
+    api_version <- default_api_version()
   }
-
+  # Check that the API version is valid
+  validate_api_version(api_version)
   # Check that the endpoint is either NULL or valid
-  is_valid_endpoint <- function(endpoint) {
-    endpoint %in% c(
-      "get-publications", "get-data-catalogue",
-      "get-summary", "get-meta",
-      "get-csv", "get-data", "post-data"
-    )
-  }
-
-  if (!is.null(endpoint)) {
-    if (is_valid_endpoint(endpoint) == FALSE) {
-      stop(
-        paste(
-          "You have entered an invalid endpoint, this should one of:",
-          "get-publications, get-data-catalogue, get-summary, get-meta,",
-          "get-csv, get-data or post-data"
-        )
-      )
-    }
-  }
+  validate_endpoint(endpoint)
 
   is_valid_dataset_info <- function(dataset_id, dataset_version) {
     !is.null(dataset_id) & (is.numeric(dataset_version) | is.null(dataset_version))
@@ -121,15 +106,6 @@ api_url <- function(
     }
   }
 
-  # Check the environment param is valid
-  if (!(environment %in% c("dev", "test", "preprod", "prod"))) {
-    stop(
-      paste(
-        "You have entered invalid EES environment. The environment should be one of:\n",
-        "   - dev, test, preprod or prod"
-      )
-    )
-  }
   # End of validation
 
   endpoint_base <- list(
@@ -140,7 +116,7 @@ api_url <- function(
   )
 
   endpoint_base_version <- paste0(
-    endpoint_base[[environment]],
+    endpoint_base[[ees_environment]],
     "v", api_version, "/"
   )
 

R/examples.R

@@ -15,7 +15,7 @@
 #'   - "filter_item" / "filter_items_short" / "filter_items_long": Return example filter ID or
 #'     example short / long filter query list.
 #'   - "indicator": Return example indicator ID
-#' @param environment Environment to return a working example for: "dev" or "test"
+#' @param ees_environment Environment to return a working example for: "dev" or "test"
 #' @param group Choose the publication group of examples to use. Options are:
 #'   - "attendance": Large example data set, careful what you ask for
 #'   - "public-api-testing": Smaller example data set
@@ -24,8 +24,9 @@
 #' @export
 #'
 #' @examples
-#' example_id("all")
 #' example_id()
+#' example_id("all")
+#' example_id("all", ees_environment = "dev")
 #' example_id("publication")
 #' example_id("publication", group = "attendance")
 #' example_id("time_period", group = "attendance")
@@ -34,8 +35,8 @@
 #' example_id("indicator", group = "attendance")
 example_id <- function(
     level = "dataset",
-    environment = "dev",
-    group = "public-api-testing") {
+    ees_environment = "test",
+    group = "absence") {
   example_id_list <- list(
     attendance = list(
       dev = list(
@@ -64,9 +65,37 @@ example_id <- function(
           reason = c("bBrtT")
         ),
         indicator = "bqZtT"
+      ),
+      test = list(
+        publication = "25d0e40b-643a-4f73-3ae5-08dcf1c4d57f",
+        dataset = "57b69201-033a-2c77-a19f-abcce2b11341",
+        time_period = "2024|W23",
+        time_periods = c("2024|W24", "2024|W25"),
+        location_id = "NAT|id|mRj9K",
+        location_ids = c("LA|id|arLPb", "REG|id|zecFQ"),
+        location_code = "NAT|code|E92000001",
+        location_codes = c("REG|code|E12000001", "REG|code|E12000002"),
+        filter = "5Zdi9",
+        filter_item = "rQkNj",
+        filter_items_long = list(
+          attendance_status = c("BfP7J", "zvUFQ"),
+          attendance_type = c("TuxPJ", "tj0Em", "5Tsdi", "fzaYF"),
+          education_phase = c("Poqeb", "dPE0Z"),
+          day_number = c("AOhGK"),
+          reason = c("9Ru4v")
+        ),
+        filter_items_short = list(
+          attendance_status = c("qGJjG"),
+          attendance_type = c("cZO31", "jgoAM"),
+          education_phase = c("Poqeb", "dPE0Z"),
+          day_number = c("AOhGK"),
+          reason = c("9Ru4v")
+        ),
+        indicator = "tj0Em",
+        indicators = c("tj0Em", "fzaYF")
       )
     ),
-    `public-api-testing` = list(
+    absence = list(
       dev = list(
         publication = "d823e4df-626f-4450-9b21-08dc8b95fc02",
         dataset = "830f9201-9e11-ad75-8dcd-d2efe2834457",
@@ -76,19 +105,30 @@ example_id <- function(
         filter = "01tT5",
         filter_item = "wEZcb",
         indicator = "PbNeb"
+      ),
+      test = list(
+        publication = "25d0e40b-643a-4f73-3ae5-08dcf1c4d57f",
+        dataset = "e1ae9201-2fff-d376-8fa3-bd3c3660d4c8",
+        location_id = "NAT|id|mRj9K",
+        location_code = "NAT|code|E92000001",
+        filter = "arLPb",
+        filter_item = "VN5XE",
+        filter_items = c("VN5XE", "PEebW"),
+        indicator = "dPe0Z",
+        indicators = c("OBXCL", "7YFXo")
       )
     )
   )
   if (!(group %in% names(example_id_list))) {
     stop(paste0("Chosen group (", group, ") not found in examples list."))
   }
-  if (!(environment %in% c("dev"))) {
-    stop(paste0("Chosen environment (", environment, ") should be one of: \"dev\"."))
+  if (!(ees_environment %in% c("dev", "test"))) {
+    stop(paste0("Chosen ees_environment (", ees_environment, ") should be one of: dev or test."))
   }
 
   group_examples <- example_id_list |>
     magrittr::extract2(group) |>
-    magrittr::extract2(environment)
+    magrittr::extract2(ees_environment)
 
   if (any(level == "all")) {
     return(group_examples)
@@ -158,15 +198,9 @@ example_data_raw <- function(
 example_json_query <- function() {
   eesyapi::parse_tojson_params(
     indicators = example_id("indicator", group = "attendance"),
-    time_periods = "2024|W23",
-    geographies = c("NAT|id|dP0Zw", "REG|id|rg3Nj"),
-    filter_items = list(
-      attendance_status = c("pmRSo"),
-      attendance_type = c("CvuId", "6AXrf"),
-      education_phase = c("ThDPJ", "crH31"),
-      day_number = c("uLQo4"),
-      reason = c("bBrtT")
-    )
+    time_periods = example_id("time_period", group = "attendance"),
+    geographies = example_id("location_codes", group = "attendance"),
+    filter_items = example_id("filter_items_short", group = "attendance")
   )
 }
 

R/get_data_catalogue.R

@@ -1,10 +1,6 @@
 #' Get publication specific data set catalogue
 #'
-#' @param publication_id The publication ID as used by the API
-#' @param page_size Number of results to collect in a single query to the API (max 40)
-#' @param page Page number to return (Default is NULL which will loop through until all
-#' pages of the query are collated).
-#' @param verbose Add extra contextual information whilst running
+#' @inheritParams api_url
 #'
 #' @return Data frame listing the data sets contained within a single publication
 #' @export
@@ -13,6 +9,8 @@
 #' get_data_catalogue(example_id("publication"))
 get_data_catalogue <- function(
     publication_id,
+    ees_environment = NULL,
+    api_version = NULL,
     page_size = NULL,
     page = NULL,
     verbose = FALSE) {
@@ -24,6 +22,8 @@ get_data_catalogue <- function(
     eesyapi::api_url(
       endpoint = "get-data-catalogue",
       publication_id = publication_id,
+      ees_environment = ees_environment,
+      api_version = api_version,
       page_size = page_size,
       page = page,
       verbose = verbose
@@ -39,6 +39,8 @@ get_data_catalogue <- function(
           eesyapi::api_url(
             endpoint = "get-data-catalogue",
             publication_id = publication_id,
+            ees_environment = ees_environment,
+            api_version = api_version,
             page_size = page_size,
             page = page,
             verbose = verbose

R/get_dataset.R

@@ -18,7 +18,7 @@
 #' @examples
 #' get_dataset(
 #'   example_id(),
-#'   geographic_levels = c("SCH"),
+#'   geographic_levels = c("NAT"),
 #'   filter_items = example_id("filter_item"),
 #'   indicators = example_id("indicator")
 #' )
@@ -30,48 +30,52 @@
     locations = NULL,
     filter_items = NULL,
     dataset_version = NULL,
+    ees_environment = NULL,
     api_version = NULL,
     page = NULL,
     page_size = 10000,
     parse = TRUE,
     verbose = FALSE) {
-  response <- eesyapi::api_url(
+  api_call <- eesyapi::api_url(
     "get-data",
     dataset_id = dataset_id,
     indicators = indicators,
     time_periods = time_periods,
     geographic_levels = geographic_levels,
     locations = locations,
     filter_items = filter_items,
+    dataset_version = dataset_version,
+    ees_environment = ees_environment,
+    api_version = api_version,
     page_size = page_size,
     page = page,
     verbose = verbose
-  ) |>
+  )
+  response <- api_call |>
     httr::GET()
   eesyapi::http_request_error(response)
   # Unless the user specifies a specific page of results to get, loop through all available pages.
   response_json <- response |>
     httr::content("text") |>
     jsonlite::fromJSON()
   if (verbose) {
-    message(paste("Total number of pages: ", response_json$paging$totalPages))
+    message(paste("Total number of pages retrieved: ", response_json$paging$totalPages))
   }
   dfresults <- response_json |>
     magrittr::extract2("results")
   # Unless the user has requested a specific page, then assume they'd like all pages collated and
   # recursively run the query.
   if (is.null(page)) {
     if (response_json$paging$totalPages > 1) {
-      if (response_json$paging$totalPages * page_size > 100000) {
-        message(
-          paste(
-            "Downloading up to", response_json$paging$totalPages * page_size, "rows.",
-            "This may take a while.",
-            "We recommend downloading the full data set using download_dataset()",
-            "for large volumes of data"
-          )
-        )
-      }
+      toggle_message(
+        paste(
+          "Downloading up to", response_json$paging$totalPages * page_size, "rows.",
+          "This may take a while.",
+          "We recommend downloading the full data set using download_dataset()",
+          "for large volumes of data"
+        ),
+        verbose = response_json$paging$totalPages * page_size > 100000
+      )
       for (page in c(2:response_json$paging$totalPages)) {
         response_page <- eesyapi::api_url(
           "get-data",
@@ -81,6 +85,9 @@
           geographic_levels = geographic_levels,
           locations = locations,
           filter_items = filter_items,
+          dataset_version = dataset_version,
+          ees_environment = ees_environment,
+          api_version = api_version,
           page_size = page_size,
           page = page,
           verbose = verbose
@@ -89,6 +96,10 @@
           httr::content("text") |>
           jsonlite::fromJSON()
         response_page |> eesyapi::warning_max_pages()
+        toggle_message(
+          paste0("Retrieved page ", page, " of ", response_json$paging$totalPages),
+          verbose = verbose
+        )
         dfresults <- dfresults |>
           dplyr::bind_rows(
             response_page |>
@@ -99,7 +110,11 @@
   }
   if (parse) {
     dfresults <- dfresults |>
-      eesyapi::parse_api_dataset(dataset_id, verbose = verbose)
+      eesyapi::parse_api_dataset(
+        dataset_id,
+        verbose = verbose,
+        ees_environment = ees_environment
+      )
   }
   return(dfresults)
 }