use explicit gdx_refprices in convGDX2MIF, ignore LDV variables with NA unit

.buildlibrary

@@ -1,10 +1,9 @@
-ValidationKey: '216806232'
+ValidationKey: '217006800'
 AcceptedWarnings:
 - 'Warning: package ''.*'' was built under R version'
 - 'Warning: namespace ''.*'' is not available and has been replaced'
 - .*qpdf.* is needed for checks on size reduction of PDFs
 - .*following variables are expected in the piamInterfaces.*
-- These variables and units contain NA
 AcceptedNotes:
 - Imports includes .* non-default packages.
 - unable to verify current time

CITATION.cff

@@ -2,8 +2,8 @@ cff-version: 1.2.0
 message: If you use this software, please cite it using the metadata from this file.
 type: software
 title: 'remind2: The REMIND R package (2nd generation)'
-version: 1.111.2
-date-released: '2023-06-03'
+version: 1.112.0
+date-released: '2023-06-07'
 abstract: Contains the REMIND-specific routines for data and model output manipulation.
 authors:
 - family-names: Rodrigues

DESCRIPTION

@@ -1,8 +1,8 @@
 Type: Package
 Package: remind2
 Title: The REMIND R package (2nd generation)
-Version: 1.111.2
-Date: 2023-06-03
+Version: 1.112.0
+Date: 2023-06-07
 Authors@R: c(
     person("Renato", "Rodrigues", , "renato.rodrigues@pik-potsdam.de", role = c("aut", "cre")),
     person("Lavinia", "Baumstark", role = "aut"),

R/colorScenConf.R

@@ -38,7 +38,7 @@ colorScenConf <- function(fileList = "", remindPath = ".", expanddata = FALSE) {
   try(cfg$gms[["regionmapping"]] <- cfg$regionmapping)
   try(cfg$gms[["inputRevision"]] <- cfg$inputRevision)
 
-  if (fileList == "") {
+  if (fileList == "" || length(fileList) == 0) {
     cat("Specify folder with .csv files. The script also searches in subdirectories.\n")
     cat(paste0("Press enter to use current: ", getwd(), ".\n"))
     folder <- getLine()

R/convGDX2MIF.R

@@ -5,26 +5,28 @@
 #'
 #'
 #' @param gdx a GDX as created by readGDX, or the file name of a gdx
-#' @param gdx_ref reference-gdx for policy costs, a GDX as created by readGDX, or the file name of a gdx
+#' @param gdx_ref reference-gdx for < cm_startyear, used for fixing the prices to this scenario
 #' @param file name of the mif file which will be written, if no name is
 #' provided a magpie object containing all the reporting information is
 #' returned
 #' @param scenario scenario name that is used in the *.mif reporting
 #' @param t temporal resolution of the reporting, default:
 #' t=c(seq(2005,2060,5),seq(2070,2110,10),2130,2150)
+#' @param gdx_refpolicycost reference-gdx for policy costs, a GDX as created by readGDX, or the file name of a gdx
 #' @author Lavinia Baumstark
 #' @examples
 #'
-#' \dontrun{convGDX2MIF(gdx,gdx_ref,file="REMIND_generic_default.csv",scenario="default")}
+#' \dontrun{convGDX2MIF(gdx,gdx_refpolicycost,file="REMIND_generic_default.csv",scenario="default")}
 #'
 #' @export
 #' @importFrom gdx readGDX
 #' @importFrom magclass mbind write.report
 
 convGDX2MIF <- function(gdx, gdx_ref = NULL, file = NULL, scenario = "default",
-                        t = c(seq(2005, 2060, 5), seq(2070, 2110, 10),
-                              2130, 2150)) {
-   # Define region subsets
+                        t = c(seq(2005, 2060, 5), seq(2070, 2110, 10), 2130, 2150),
+                        gdx_refpolicycost = gdx_ref) {
+
+  # Define region subsets
   regionSubsetList <- toolRegionSubsets(gdx)
   # ADD EU-27 region aggregation if possible
   if("EUR" %in% names(regionSubsetList)){
@@ -80,22 +82,22 @@ convGDX2MIF <- function(gdx, gdx_ref = NULL, file = NULL, scenario = "default",
   output <- mbind(output,reportCrossVariables(gdx,output,regionSubsetList,t)[,t,])
 
   # Report policy costs, if possible and sensible
-  if(!is.null(gdx_ref)) {
-    if (file.exists(gdx_ref)) {
+  if(!is.null(gdx_refpolicycost)) {
+    if (file.exists(gdx_refpolicycost)) {
       gdp_scen <- try(readGDX(gdx,"cm_GDPscen",react ="error"),silent=T)
-      gdp_scen_ref <- try(readGDX(gdx_ref,"cm_GDPscen",react = "error"),silent=T)
+      gdp_scen_ref <- try(readGDX(gdx_refpolicycost,"cm_GDPscen",react = "error"),silent=T)
       if(!inherits(gdp_scen,"try-error") && !inherits(gdp_scen_ref,"try-error")){
         if(gdp_scen[1]==gdp_scen_ref[1]){
-          message("running reportPolicyCosts, comparing to ", basename(dirname(gdx_ref)), "/", basename(gdx_ref), "...")
-          output <- mbind(output,reportPolicyCosts(gdx,gdx_ref,regionSubsetList,t)[,t,])
+          message("running reportPolicyCosts, comparing to ", basename(dirname(gdx_refpolicycost)), "/", basename(gdx_refpolicycost), "...")
+          output <- mbind(output,reportPolicyCosts(gdx,gdx_refpolicycost,regionSubsetList,t)[,t,])
         } else {
           warning(paste0("The GDP scenario differs from that of the reference run. Did not execute 'reportPolicyCosts'! If a policy costs reporting is desired, please use the 'policyCosts' output.R script."))
         }
       } else {
         warning(paste0("A comparison of the GDP scenarios between this run and its reference run wasn't possible (old remind version). Therefore to avoid reporting unsensible policy costs, 'reportPolicyCosts' was not executed. If a policy costs reporting is required, please use the  'policyCosts' output.R script."))
       }
     } else {
-      warning(paste0("File ",gdx_ref," not found. Did not execute 'reportPolicyCosts'! If a policy costs reporting is desired, please use the   'policyCosts' output.R script."))
+      warning(paste0("File ",gdx_refpolicycost," not found. Did not execute 'reportPolicyCosts'! If a policy costs reporting is desired, please use the   'policyCosts' output.R script."))
     }
   }
 

R/reportPrices.R

@@ -691,15 +691,17 @@ reportPrices <- function(gdx, output=NULL, regionSubsetList=NULL,
   out.reporting <- pmax(out, 0) # avoid negative prices
 
   # for cm_startyear and non-SSP2, replace price by average of period before and after
+  # this is a workaround to avoid spikes caused by https://github.com/remindmodel/remind/issues/1068
   if (! grepl("gdp_SSP2", readGDX(gdx, "cm_GDPscen", format = "simplest"))
       && cm_startyear > min(getYears(out, as.integer = TRUE))) {
     out.reporting[, cm_startyear, ] <- 0.5 * (out[, cm_startyear - 5, ] + out[, cm_startyear + 5, ])
   }
   out.reporting <- lowpass(out.reporting)
   # reset values for years smaller than cm_startyear to avoid inconsistencies in cm_startyear - 5
   if (! is.null(gdx_ref)) {
+    message("reportPrices loads price for < cm_startyear from gdx_ref.")
     priceRef <- try(reportPrices(gdx_ref, output = NULL, regionSubsetList = regionSubsetList, t = t))
-    fixedyears <- getYears(out)[getYears(out, as.integer = TRUE) < as.integer(cm_startyear)]
+    fixedyears <- getYears(out)[getYears(out, as.integer = TRUE) < cm_startyear]
     if (! inherits(priceRef, "try-error") && length(fixedyears) > 0) {
       out.reporting[, fixedyears, ] <- priceRef[getRegions(out), fixedyears, getNames(out)]
     }
@@ -918,12 +920,22 @@ reportPrices <- function(gdx, output=NULL, regionSubsetList=NULL,
                "Price|Final Energy|Industry|Solids (US$2005/GJ)"       = "FE|Industry|Solids (EJ/yr)"
                )
 
+  # transport-specific mappings depending on realization
+  if (module2realisation["transport",2] == "complex") {
+    int2ext <- c(int2ext,
+                 "Price|Final Energy|Transport|Liquids|HDV (US$2005/GJ)"       = "FE|Transport|non-LDV|Liquids (EJ/yr)",
+                 "Price|Final Energy|Transport|Liquids|LDV (US$2005/GJ)"       = "FE|Transport|LDV|Liquids (EJ/yr)")
+  } else if (module2realisation["transport",2] == "edge_esm") {
+    int2ext <- c(int2ext,
+                 "Price|Final Energy|Transport|Liquids|HDV (US$2005/GJ)"       = "FE|Transport|Diesel Liquids (EJ/yr)",
+                 "Price|Final Energy|Transport|Liquids|LDV (US$2005/GJ)"       = "FE|Transport|Pass|Liquids (EJ/yr)")
+  }
 
-  ## weights definition for FE prices region aggregation
+  ## add weights definition for region aggregation for FE prices that were added automatically
   if(length(pm_FEPrice_by_FE) > 0) {
-    margPriceVars <- getItems(out,3)[grep("Price|Final Energy|", getItems(out,3), fixed = TRUE)]
+    margPriceVars <- grep("Price|Final Energy|", getItems(out,3), fixed = TRUE, value = TRUE)
     margPriceVars <- setdiff(margPriceVars, names(int2ext))
-    vars <- gsub("US\\$2005/GJ","EJ/yr",gsub("Price\\|Final Energy\\|","FE|",margPriceVars))
+    vars <- gsub("US\\$2005/GJ", "EJ/yr", gsub("Price\\|Final Energy\\|","FE|",margPriceVars))
     names(vars) <- margPriceVars
     vars <- gsub("Efuel","Hydrogen",vars) ###warning FE variable should be renamed and this line should be removed in the future
     # for(var in vars){ # display price variables with no matching FE weight
@@ -935,21 +947,6 @@ reportPrices <- function(gdx, output=NULL, regionSubsetList=NULL,
     int2ext <- c(int2ext, vars)
   }
 
-  # transport-specific mappings depending on realization
-
-  if (module2realisation["transport",2] == "complex") {
-    int2ext <- c(int2ext,
-                 "Price|Final Energy|Transport|Liquids|HDV (US$2005/GJ)"       = "FE|Transport|non-LDV|Liquids (EJ/yr)",
-                 "Price|Final Energy|Transport|Liquids|LDV (US$2005/GJ)"       = "FE|Transport|LDV|Liquids (EJ/yr)")
-  } else if (module2realisation["transport",2] == "edge_esm") {
-    int2ext <- c(int2ext,
-                 "Price|Final Energy|Transport|Liquids|HDV (US$2005/GJ)"       = "FE|Transport|Diesel Liquids (EJ/yr)",
-                 "Price|Final Energy|Transport|Liquids|LDV (US$2005/GJ)"       = "FE|Transport|Pass|Liquids (EJ/yr)",
-                 "Price|Final Energy|Transport|Gases (US$2005/GJ)"       = "FE|Transport|Gases (EJ/yr)")
-  }
-
-
-
   ## moving averages and rawdata
   avgs <- getNames(out.lowpass)
   rawdata <- getNames(out.rawdata)

README.md

@@ -1,6 +1,6 @@
 # The REMIND R package (2nd generation)
 
-R package **remind2**, version **1.111.2**
+R package **remind2**, version **1.112.0**
 
 [![CRAN status](https://www.r-pkg.org/badges/version/remind2)](https://cran.r-project.org/package=remind2)  [![R build status](https://github.com/pik-piam/remind2/workflows/check/badge.svg)](https://github.com/pik-piam/remind2/actions) [![codecov](https://codecov.io/gh/pik-piam/remind2/branch/master/graph/badge.svg)](https://app.codecov.io/gh/pik-piam/remind2) [![r-universe](https://pik-piam.r-universe.dev/badges/remind2)](https://pik-piam.r-universe.dev/builds)
 
@@ -49,7 +49,7 @@ In case of questions / problems please contact Renato Rodrigues <renato.rodrigue
 
 To cite package **remind2** in publications use:
 
-Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P (2023). _remind2: The REMIND R package (2nd generation)_. R package version 1.111.2, <https://github.com/pik-piam/remind2>.
+Rodrigues R, Baumstark L, Benke F, Dietrich J, Dirnaichner A, Führlich P, Giannousakis A, Hasse R, Hilaire J, Klein D, Koch J, Kowalczyk K, Levesque A, Malik A, Merfort A, Merfort L, Morena-Leiva S, Pehl M, Pietzcker R, Rauner S, Richters O, Rottoli M, Schötz C, Schreyer F, Siala K, Sörgel B, Spahr M, Strefler J, Verpoort P (2023). _remind2: The REMIND R package (2nd generation)_. R package version 1.112.0, <URL: https://github.com/pik-piam/remind2>.
 
 A BibTeX entry for LaTeX users is
 
@@ -58,7 +58,7 @@ A BibTeX entry for LaTeX users is
   title = {remind2: The REMIND R package (2nd generation)},
   author = {Renato Rodrigues and Lavinia Baumstark and Falk Benke and Jan Philipp Dietrich and Alois Dirnaichner and Pascal Führlich and Anastasis Giannousakis and Robin Hasse and Jérome Hilaire and David Klein and Johannes Koch and Katarzyna Kowalczyk and Antoine Levesque and Aman Malik and Anne Merfort and Leon Merfort and Simón Morena-Leiva and Michaja Pehl and Robert Pietzcker and Sebastian Rauner and Oliver Richters and Marianna Rottoli and Christof Schötz and Felix Schreyer and Kais Siala and Björn Sörgel and Mike Spahr and Jessica Strefler and Philipp Verpoort},
   year = {2023},
-  note = {R package version 1.111.2},
+  note = {R package version 1.112.0},
   url = {https://github.com/pik-piam/remind2},
 }
 ```

tests/testthat/test-convGDX2mif.R

@@ -42,6 +42,7 @@ checkIntegrity <- function(out, gdxPath = NULL) {
     warning("These variable names have wrong bars and spaces: ", paste(barspace, collapse = ", "))
   }
   NAvar <- grep("[\\|\\( ]NA[\\|\\) ]|^NA", unique(dt[["variable"]]), value = TRUE)
+  NAvar <- NAvar[! grepl("^Services and Products\\|Transport\\|non-LDV\\|S", NAvar)] # unit NA, but ok, see issue #408
   if (length(NAvar) > 0) {
     warning("These variables and units contain NA: ", paste(NAvar, collapse = ", "))
   }
@@ -115,7 +116,7 @@ test_that("Test if REMIND reporting is produced as it should and check data inte
   for (gdxPath in gdxPaths) {
     numberOfMifs <- numberOfMifs + 1
     message("Running convGDX2MIF(", gdxPath, ")...")
-    mifContent <- convGDX2MIF(gdxPath, gdx_ref = gdxPath)
+    mifContent <- convGDX2MIF(gdxPath, gdx_refpolicycost = gdxPath)
     message("Checking integrity of created MIF...")
     checkIntegrity(mifContent, gdxPath)
     magclass::write.report(

tests/testthat/test-piamInterfaces-Ariadne.R

@@ -10,7 +10,7 @@ test_that("Test if REMIND reporting produces mandatory variables for Ariadne rep
                        mode = "wb", quiet = TRUE
   )
 
-  mif <- convGDX2MIF(gdxPath, gdx_ref = gdxPath)
+  mif <- convGDX2MIF(gdxPath, gdx_refpolicycost = gdxPath)
 
   computedVariables <- getItems(mif, dim = 3.3)
 

tests/testthat/test-piamInterfaces-NAVIGATE.R

@@ -10,7 +10,7 @@ test_that("Test if REMIND reporting produces mandatory variables for NAVIGATE re
                        mode = "wb", quiet = TRUE
   )
 
-  mif <- convGDX2MIF(gdxPath, gdx_ref = gdxPath)
+  mif <- convGDX2MIF(gdxPath, gdx_refpolicycost = gdxPath)
 
   computedVariables <- getItems(mif, dim = 3.3)
 

tests/testthat/test-piamInterfaces-NGFS.R

@@ -9,7 +9,7 @@ test_that("Test if REMIND reporting produces mandatory variables for NGFS report
     mode = "wb", quiet = TRUE
   )
 
-  mif <- convGDX2MIF(gdxPath, gdx_ref = gdxPath)
+  mif <- convGDX2MIF(gdxPath, gdx_refpolicycost = gdxPath)
 
   computedVariables <- getItems(mif, dim = 3.3)
 

tests/testthat/test-piamInterfaces-SHAPE.R

@@ -10,7 +10,7 @@ test_that("Test if REMIND reporting produces mandatory variables for SHAPE repor
                        mode = "wb", quiet = TRUE
   )
 
-  mif <- convGDX2MIF(gdxPath, gdx_ref = gdxPath)
+  mif <- convGDX2MIF(gdxPath, gdx_refpolicycost = gdxPath)
 
   computedVariables <- getItems(mif, dim = 3.3)