Merge branch 'feature/cran_resubmission' into devel

.Rbuildignore

@@ -15,8 +15,15 @@
 ^pkgdown$
 ^codecov\.yml$
 ^utils
-^vignettes/faq\.Rmd$
-^vignettes/discovered_bug\.Rmd$
-^vignettes/lorenzo\.Rmd$
-^vignettes/MODIStsp\.Rmd$
+^vignettes/analyze\.Rmd
+^vignettes/discovered_bug\.Rmd
+^vignettes/examples\.Rmd
+^vignettes/faq\.Rmd
+^vignettes/installation\.Rmd
+^vignettes/interactive_execution\.Rmd
+^vignettes/lorenzo\.Rmd
+^vignettes/noninteractive_execution\.Rmd
+^vignettes/output\.Rmd
+^vignettes/products_list\.Rmd
+^vignettes/standalone_execution\.Rmd
 ^CONDUCT\.md$

DESCRIPTION

@@ -1,5 +1,5 @@
 Package: MODIStsp
-Title: A Tool for Automating Download and Preprocessing of MODIS Land Products
+Title: Find, Download and Process MODIS Land Products
     Data
 Type: Package
 Version: 2.0.5
@@ -17,22 +17,28 @@ Authors@R: c(person("Lorenzo", "Busetto",
              person("Jeff", "Hanson", 
                  role = "rev",
                  comment = "Jeff Hanson reviewed the package for rOpenSci, see 
-                     https://github.com/ropensci/software-review/issues/184"))
+                     https://github.com/ropensci/software-review/issues/184"),
+             person("Babak", "Naimi", 
+                 role = "ctb",
+                 comment = "Babak Naimi wrote the function ModisDownload(),
+                     on which some MODIStsp internal functions are based"))
 Description: Allows automating the creation of time series of rasters derived
-    from MODIS Satellite Land Products data. It performs several typical
-    preprocessing steps such as download, mosaicking, reprojection and resize
-    of data acquired on a specified time period. All processing parameters
+    from MODIS satellite land products data. It performs several typical
+    preprocessing steps such as download, mosaicking, reprojecting and resizing
+    data acquired on a specified time period. All processing parameters
     can be set using a user-friendly GUI. Users can select which layers of
     the original MODIS HDF files they want to process, which additional
-    Quality Indicators should be extracted from aggregated MODIS Quality
-    Assurance layers and, in the case of Surface Reflectance products
-    , which Spectral Indexes should be computed from the original reflectance
+    quality indicators should be extracted from aggregated MODIS quality
+    assurance layers and, in the case of surface reflectance products,
+    which spectral indexes should be computed from the original reflectance
     bands. For each output layer, outputs are saved as single-band raster
     files corresponding to each available acquisition date. Virtual files
     allowing access to the entire time series as a single file are also created.
     Command-line execution exploiting a previously saved processing options
-    file is also possible, allowing to automatically update time series
+    file is also possible, allowing users to automatically update time series
     related to a MODIS product whenever a new image is available.
+    For additional documentation refer to the following article: 
+    Busetto and Ranghetti (2016) <doi:10.1016/j.cageo.2016.08.020>.
 License: GPL-3
 Depends:
     R (>= 3.5.0)
@@ -63,9 +69,7 @@ Suggests:
     leaflet,
     magrittr,
     mapedit (>= 0.6.0),
-    markdown,
     png,
-    qpdf,
     rappdirs,
     rgdal, 
     rmarkdown,
@@ -77,7 +81,6 @@ Suggests:
     spelling,
     testthat, 
     tibble,
-    tibbletime,
     tidyr,
     xtable
 SystemRequirements: 

NEWS.md

@@ -5,6 +5,9 @@
 - Edit documentation related to the change of maintainer
     (see https://docs.ropensci.org/MODIStsp/articles/lorenzo).
 
+- Add the argument `parallel` to function `MODIStsp()` and `MODSIStsp_process()`
+    to allow running the processing in single core modality.
+
 ### Minor changes
 
 - Fix Travis tests

R/MODIStsp.R

@@ -72,7 +72,7 @@
 #'  to use to derive the processing extent. If not NULL, the processing options
 #'  which define the extent, the selected tiles and the "Full Tile / Custom"
 #'  in the JSON options file are overwritten and new files are created on the
-#'  extent of the provided spatial file. Ignored if spatmet != "file", Default: NULL
+#'  extent of the provided spatial file. Ignored if spatmeth != "file", Default: NULL
 #' @param out_projsel `character ["Native", "User Defined`] If "Native", the
 #'   outputs keep the original resolution of MODIS HDF images. Otherwise, the value
 #'    set in "out_res" is used, Default:Native
@@ -110,6 +110,12 @@
 #'   MODIStsp_process will abort, Default: 20
 #' @param verbose `logical` If FALSE, suppress processing messages,
 #'  Default: TRUE
+#' @param parallel `logical` If TRUE (default), the function is run using parallel
+#'  processing, to speed-up the computation for large rasters (with a maximum
+#'  of 8 cores).
+#'  The number of cores is automatically determined; specifying it is also 
+#'  possible (e.g. `parallel = 4`). In this case, more than 8 cores can be
+#'  specified. If FALSE (default), single core processing is used.
 #' @param ... not used for values, forces later arguments to bind by name
 #' @return NULL
 #'
@@ -126,15 +132,15 @@
 #' @importFrom utils unzip
 #' @examples
 #'
-#' \dontrun{
 #' #' # - Running the tool using the GUI
-#'
 #' # Running the tool without any option will start the GUI with the default or
 #' # last used settings, in interactive mode (i.e., with gui = TRUE).
-#'
-#' # MODIStsp()
+#' \donttest{
+#' if (interactive()) {
+#'   MODIStsp()
 #' }
-#' \dontrun{
+#' }
+#'
 #'
 #' #' # - Running the tool specifying processing arguments in the call
 #'
@@ -143,25 +149,29 @@
 #' # Here we process layers __NDVI__ and __EVI__ and quality indicator __usefulness__
 #' # of product __M*D13Q1__, considering both Terra and Aqua platforms, for dates
 #' # comprised between 2020-06-01 and 2020-06-15 and saves output to R tempdir
-#' # --> See name and available layers for product M*D13Q1
+#' # --> See name and available layers for product M*D13Q1.
+#' # Note that this example (as well as the following ones) is run in single
+#' # core to follow CRAN policies, by setting parallel = FALSE.
+#' # Users can exploit multicore functionalities skipping to set this argument.
 #'
 #' MODIStsp_get_prodlayers("M*D13A2")
-#'
-#' # --> Launch the processing
-#' MODIStsp(gui             = FALSE,
-#'          out_folder      = "$tempdir",
-#'          selprod         = "Vegetation_Indexes_16Days_1Km (M*D13A2)",
-#'          bandsel         = c("EVI", "NDVI"),
-#'          quality_bandsel = "QA_usef",
-#'          indexes_bandsel = "SR",
-#'          user            = "mstp_test" ,
-#'          password        = "MSTP_test_01",
-#'          start_date      = "2020.06.01",
-#'          end_date        = "2020.06.15",
-#'          verbose         = FALSE)
+#' \donttest{
+#' MODIStsp(
+#'   gui = FALSE,
+#'   out_folder = "$tempdir",
+#'   selprod = "Vegetation_Indexes_16Days_1Km (M*D13A2)",
+#'   bandsel = c("EVI", "NDVI"),
+#'   quality_bandsel = "QA_usef",
+#'   indexes_bandsel = "SR",
+#'   user = "mstp_test" ,
+#'   password = "MSTP_test_01",
+#'   start_date = "2020.06.01",
+#'   end_date = "2020.06.15",
+#'   verbose = FALSE,
+#'   parallel = FALSE
+#' )
 #' }
 #'
-#' \dontrun{
 #'
 #' #' # - Running the tool using the settings previously saved in a specific options file
 #'
@@ -173,62 +183,73 @@
 #' # and retrieves NDVI and EVI data, plus the Usefulness Index Quality Indicator.
 #'
 #' opts_file <- system.file("testdata/test_MOD13A2.json", package = "MODIStsp")
-#' MODIStsp(gui = FALSE, opts_file = opts_file, verbose = TRUE)
+#' \donttest{
+#' MODIStsp(gui = FALSE, opts_file = opts_file, verbose = TRUE, parallel = FALSE)
 #' }
 #'
-#' \dontrun{
 #'
 #' # Running the tool using the settings previously saved in a specific option file
 #' # and specifying the extent from a spatial file allows to re-use the same
 #' # processing settings to perform download and reprocessing on a different area
 #'
 #' opts_file <- system.file("testdata/test_MOD13A2.json", package = "MODIStsp")
 #' spatial_file <- system.file("testdata/lakeshapes/garda_lake.shp", package = "MODIStsp")
-#' MODIStsp(gui = FALSE, opts_file = opts_file,
+#' \donttest{
+#' MODIStsp(
+#'   gui = FALSE, 
+#'   opts_file = opts_file,
 #'   spatmeth = "file",
-#'   spafile = spatial_file, verbose = TRUE)
+#'   spafile = spatial_file, 
+#'   verbose = TRUE,
+#'   parallel = FALSE
+#' )
 #' }
 #'
-#' \dontrun{
 #'
 #' # Running the tool using the settings previously saved in a
 #' # specific options file and specifying each time the extent from a different
 #' # spatial file (e.g., to perform the same processing on several extents)
-#'
-#' extent_list  <- c(system.file("testdata/lakeshapes/garda_lake.shp",
-#'                               package = "MODIStsp"),
-#'                   system.file("testdata/lakeshapes/iseo_lake.shp",
-#'                               package = "MODIStsp"))
-#' extent_list
-#'
 #' # Note that you can also put all your extent files in a specific folder and
 #' # create the extent list using for example.
-#' # extent_list = list.files(system.file("testdata/lakeshapes/", package = "MODIStsp"),
-#' #                          full.names = TRUE, "\\.shp$")
-#'
+#' 
+#' extent_list = list.files(
+#'   system.file("testdata/lakeshapes/", package = "MODIStsp"),
+#'   "\\.shp$", 
+#'   full.names = TRUE
+#' )
+#' extent_list
 #' opts_file <- system.file("testdata/test_MOD13A2.json", package = "MODIStsp")
+#' 
+#' \donttest{
 #' for (single_shape in extent_list) {
-#'   MODIStsp(gui = FALSE, opts_file = opts_file,
-#'            spatmeth = "file",
-#'            spafile = single_shape, verbose = TRUE)
+#'   MODIStsp(
+#'     gui = FALSE, 
+#'     opts_file = opts_file,
+#'     spatmeth = "file",
+#'     spafile = single_shape, 
+#'     verbose = TRUE,
+#'     parallel = FALSE
+#'  )
 #' }
 #'
 #' # output files are placed in separate folders:
-#' outfiles_garda <- list.files(file.path(tempdir(), "MODIStsp/garda_lake/VI_16Days_1Km_v6/NDVI"),
-#'            full.names = TRUE)
+#' outfiles_garda <- list.files(
+#'   file.path(tempdir(), "MODIStsp/garda_lake/VI_16Days_1Km_v6/NDVI"),
+#'   full.names = TRUE
+#' )
 #' outfiles_garda
-#'
 #' library(raster)
 #' plot(raster(outfiles_garda[1] ))
 #'
-#' outfiles_iseo <- list.files(file.path(tempdir(), "MODIStsp/iseo_lake/VI_16Days_1Km_v6/NDVI"),
-#'            full.names = TRUE)
+#' outfiles_iseo <- list.files(
+#'   file.path(tempdir(), "MODIStsp/iseo_lake/VI_16Days_1Km_v6/NDVI"),
+#'   full.names = TRUE
+#' )
 #' outfiles_iseo
-#'
 #' plot(raster(outfiles_iseo[1]))
+#' }
 #'
 #' # See also https://docs.ropensci.org/MODIStsp/articles/noninteractive_execution.html
-#' }
 
 MODIStsp <- function(...,
                      gui             = TRUE,
@@ -268,7 +289,8 @@ MODIStsp <- function(...,
                      compress        = NULL,
                      test            = NULL,
                      n_retries       = 5,
-                     verbose         = TRUE) {
+                     verbose         = TRUE,
+                     parallel        = TRUE) {
 
   # Make so that "raster" functions does not automatically add extensions on
   # output files. This is automatically reset to TRUE at the end of the session
@@ -540,7 +562,8 @@ MODIStsp <- function(...,
 
     MODIStsp_process(proc_opts,
                      n_retries = n_retries,
-                     verbose = verbose)
+                     verbose = verbose,
+                     parallel = parallel)
   }
 
 }

R/MODIStsp_process.R

@@ -23,6 +23,12 @@
 #'   In case any download function fails more than `n_retries` times consecutively,
 #'   MODIStsp_process will abort, Default: 20
 #' @param verbose `logical` If FALSE, suppress processing messages, Default: TRUE
+#' @param parallel `logical` If TRUE (default), the function is run using parallel
+#'  processing, to speed-up the computation for large rasters (with a maximum
+#'  of 8 cores).
+#'  The number of cores is automatically determined; specifying it is also 
+#'  possible (e.g. `parallel = 4`). In this case, more than 8 cores can be
+#'  specified. If FALSE (default), single core processing is used.
 #' @return The function is called for its side effects.
 #' @author Lorenzo Busetto, phD (2014-2017)
 #' @author Luigi Ranghetti, phD (2015) \email{luigi@@ranghetti.info}
@@ -36,7 +42,8 @@
 
 MODIStsp_process <- function(proc_opts,
                              n_retries,
-                             verbose = TRUE) {
+                             verbose = TRUE,
+                             parallel = TRUE) {
 
   # Based on sel_prod, retrieve needed variables from prod_opts file
 
@@ -425,7 +432,7 @@ MODIStsp_process <- function(proc_opts,
                     proc_opts$out_format, outrep_file, proc_opts$compress,
                     proc_opts$out_res_sel, proc_opts$out_res, proc_opts$resampling,
                     proc_opts$nodata_change,
-                    gui, verbose
+                    gui, verbose, parallel
                   )
                 }
               }	# END Cycle on available MODIS Bands

R/MODIStsp_process_QA_bits.R

@@ -23,7 +23,7 @@
 #' @author Lorenzo Busetto, phD (2017)
 #' @author Luigi Ranghetti, phD (2017) \email{luigi@@ranghetti.info}
 #' @note License: GPL 3.0
-#'  Based on the "modis.qc.R" script by Yann Chemin (2008) (https://goo.gl/7Fhreo)
+#'  Based on the `modis.qc.R` script by Yann Chemin (2008) (\url{https://goo.gl/7Fhreo})
 #'  license GPL 3.0
 #' @importFrom raster raster NAvalue calc
 #' @importFrom bitops bitShiftR bitAnd

R/MODIStsp_process_bands.R

@@ -29,6 +29,12 @@
 #' @param outrep_file `character` Full path of the file where results of the
 #'  processing are to be stored (created in `MODIStsp_process`)
 #' @param verbose `logical` If FALSE, suppress processing messages, Default: TRUE
+#' @param parallel `logical` If TRUE, the function is run using parallel
+#'  processing, to speed-up the computation for large rasters (with a maximum
+#'  of 8 cores).
+#'  The number of cores is automatically determined; specifying it is also 
+#'  possible (e.g. `parallel = 4`). In this case, more than 8 cores can be
+#'  specified. If FALSE (default), single core processing is used.
 #' @return The function is called for its side effects
 #' @rdname MODIStsp_process_bands
 #' @author Lorenzo Busetto, phD (2014-2017)
@@ -50,7 +56,7 @@ MODIStsp_process_bands <- function(out_folder_mod, modislist,
                                    out_format, outrep_file, compress,
                                    out_res_sel, out_res, resampling,
                                    nodata_change,
-                                   gui, verbose) {
+                                   gui, verbose, parallel) {
 
   tmpdir <- file.path(tempdir(), "mstp_temp")
   dir.create(tmpdir, showWarnings = FALSE)
@@ -62,7 +68,13 @@ MODIStsp_process_bands <- function(out_folder_mod, modislist,
   #   number of available cpus > 8 (this to avoid overloading servers with high
   #   number of cpus)
 
-  ncores <- min(c(8, parallel::detectCores() - 2))
+  ncores <- if (is.numeric(parallel)) {
+    as.integer(parallel)
+  } else if (parallel == FALSE) {
+    1
+  } else {
+    min(c(8, parallel::detectCores() - 2))
+  }
 
   # check to see if the patch to correct wrong resolution/bbox in some HDF4 ----
   # original layers (e.g. albedo) is needed
@@ -316,7 +328,7 @@ MODIStsp_process_bands <- function(out_folder_mod, modislist,
                             outproj_str,
                             mod_proj_str,
                             enlarge = TRUE)
-    # Create a resized and (if needed) mosaiced GDAL vrt file
+    # Create a resized and (if needed) mosaicked GDAL vrt file
 
     if (datatype == "UInt32") {
       # fix to avoid bug on gdalbuildvrt for UInt32 datasets;

R/get_mod_dirs.R

@@ -16,7 +16,7 @@
 #'   environment or not. If not, processing messages are sent to a log file
 #'   instead than to the console/GTK progress windows.
 #' @param out_folder_mod  `character` output folder for MODIS HDF storage
-#' @return `character arraay` listing all available folders (a.k.a. dates) for
+#' @return `character array` listing all available folders (a.k.a. dates) for
 #'   the requested MODIS product on lpdaac http archive, for the years
 #'   included in the time range selected for processing.
 #' @author Original code by Babak Naimi (\code{.getModisList}, in