Merge pull request #1370 from orichters/fix_bau_checker

Fix check if path_gdx_bau is correctly set (NA for no NDC run, not NA for NDC)

main.gms

@@ -1621,17 +1621,17 @@ $setGlobal cm_conoptv  conopt3    !! def = conopt3
 *'
 *' (off): normal model operation, default
 *' (on): no model operation, instead input.gdx is copied to fulldata.gdx
-$setGlobal c_empty_model   off    !! def = off
+$setGlobal c_empty_model   off    !! def = off  !! regexp = off|on
 *' mode for solving nash problem
 *'
 *' * parallel  - all regions are run an parallel
 *' * debug     - all regions are run in a sequence and the lst-file will contain information on infeasiblities
-$setGlobal cm_nash_mode  parallel      !! def = parallel
+$setGlobal cm_nash_mode  parallel      !! def = parallel  !! regexp = debug|parallel|serial
 
 $setglobal cm_secondary_steel_bound  scenario   !! def = scenario
 $setglobal c_GDPpcScen  SSP2EU     !! def = gdp_SSP2   (automatically adjusted by start_run() based on GDPscen)
 $setglobal cm_demScen  gdp_SSP2EU     !! def = gdp_SSP2EU
-$setGlobal c_scaleEmiHistorical  on  !! def = on
+$setGlobal c_scaleEmiHistorical  on  !! def = on  !! regexp = off|on
 $SetGlobal cm_quick_mode  off          !! def = off  !! regexp = off|on
 $setGLobal cm_debug_preloop  off    !! def = off  !! regexp = off|on
 $setGlobal cm_APscen  SSP2          !! def = SSP2

scripts/start/readCheckScenarioConfig.R

@@ -34,6 +34,10 @@ readCheckScenarioConfig <- function(filename, remindPath = ".", testmode = FALSE
             paste0(rownames(scenConf)[regionname], collapse = ", "),
             " – Titles with three capital letters or 'glob' may be confused with region names by magclass. Stopping now.")
   }
+  nameisNA <- grepl("^NA$", rownames(scenConf))
+  if (any(nameisNA)) {
+    warning("Don't use 'NA' as scenario name, you fool. Stopping now.")
+  }
   illegalchars <- grepl("[^[:alnum:]_-]", rownames(scenConf))
   if (any(illegalchars)) {
     warning("These titles contain illegal characters: ",
@@ -51,12 +55,6 @@ readCheckScenarioConfig <- function(filename, remindPath = ".", testmode = FALSE
       whitespaceErrors <- whitespaceErrors + sum(haswhitespace)
     }
   }
-  if ("path_gdx_ref" %in% names(scenConf) && ! "path_gdx_refpolicycost" %in% names(scenConf)) {
-    scenConf$path_gdx_refpolicycost <- scenConf$path_gdx_ref
-    message("In ", filename,
-        ", no column path_gdx_refpolicycost for policy cost comparison found, using path_gdx_ref instead.")
-  }
-  scenConf[, names(path_gdx_list)[! names(path_gdx_list) %in% names(scenConf)]] <- NA
 
   # fill empty cells with values from scenario written in copyConfigFrom cell
   copyConfigFromErrors <- 0
@@ -72,7 +70,38 @@ readCheckScenarioConfig <- function(filename, remindPath = ".", testmode = FALSE
     }
   }
 
-  errorsfound <- sum(toolong) + sum(regionname) + sum(illegalchars) + whitespaceErrors + copyConfigFromErrors
+  pathgdxerrors <- 0
+  # fix missing path_gdx and inconsistencies
+  if ("path_gdx_ref" %in% names(scenConf) && ! "path_gdx_refpolicycost" %in% names(scenConf)) {
+    scenConf$path_gdx_refpolicycost <- scenConf$path_gdx_ref
+    msg <- paste0("In ", filename,
+        ", no column path_gdx_refpolicycost for policy cost comparison found, using path_gdx_ref instead.")
+    message(msg)
+  }
+  if ("path_gdx_bau" %in% names(scenConf)) {
+    # fix if bau given despite not needed
+    NDC45 <- if ("carbonprice" %in% names(scenConf)) scenConf$carbonprice %in% "NDC" else FALSE
+    NDC46 <- if ("carbonpriceRegi" %in% names(scenConf)) scenConf$carbonpriceRegi %in% "NDC" else FALSE
+    noNDCbutBAU <- ! is.na(scenConf$path_gdx_bau) & ! (NDC45 | NDC46)
+    if (sum(noNDCbutBAU) > 0) {
+      msg <- paste0("In ", sum(noNDCbutBAU), " scenarios, neither 'carbonprice' nor 'carbonpriceRegi' is set to 'NDC', but 'path_gdx_bau' is not empty.\n",
+                    "To avoid unnecessary dependencies to other runs, setting 'path_gdx_bau' to NA.")
+      message(msg)
+      scenConf$path_gdx_bau[noNDCbutBAU] <- NA
+    }
+    # fail if bau not given but needed
+    noBAUbutNDC <- is.na(scenConf$path_gdx_bau) & (NDC45 | NDC46)
+    if (sum(noBAUbutNDC) > 0) {
+      pathgdxerrors <- pathgdxerrors + sum(noBAUbutNDC)
+      warning("In ", sum(noBAUbutNDC), " scenarios, 'carbonprice' or 'carbonpriceRegi' is set to 'NDC' ",
+              "which requires a reference gdx in 'path_gdx_bau', but it is empty.")
+    }
+  }
+  # make sure every path gdx column exists
+  scenConf[, names(path_gdx_list)[! names(path_gdx_list) %in% names(scenConf)]] <- NA
+
+  # collect errors
+  errorsfound <- sum(toolong) + sum(regionname) + sum(nameisNA) + sum(illegalchars) + whitespaceErrors + copyConfigFromErrors + pathgdxerrors
 
   # check column names
   knownColumnNames <- c(names(cfg$gms), names(path_gdx_list), "start", "output", "description", "model",

scripts/start/run.R

@@ -199,6 +199,10 @@ run <- function(start_subsequent_runs = TRUE) {
 
       gdx_na <- is.na(cfg$files2export$start[pathes_to_gdx])
       needfulldatagdx <- names(cfg$files2export$start[pathes_to_gdx][cfg$files2export$start[pathes_to_gdx] == cfg_main$title & !gdx_na])
+      if (length(needfulldatagdx) == 0) {
+        message("Somehow, my gdx file was not needed although cfg$RunsUsingTHISgxAsInput expected that. Skipping ", run)
+        next
+      }
       message("In ", RData_file, ", use current fulldata.gdx path for ", paste(needfulldatagdx, collapse = ", "), ".")
       cfg$files2export$start[needfulldatagdx] <- fulldatapath
 

scripts/start/submit.R

@@ -75,7 +75,7 @@ submit <- function(cfg, restart = FALSE, stopOnFolderCreateError = TRUE) {
         message("done.")
       } else {
         # a run renv is loaded, we are presumably starting new run in a cascade
-        message("Copying lockfile into '", cfg$results_folder, "'")
+        message("   Copying lockfile into '", cfg$results_folder, "'")
         file.copy(renv::paths$lockfile(), file.path(cfg$results_folder, "_renv.lock"))
       }
 

start.R

@@ -158,7 +158,7 @@ if (any(c("--reprepare", "--restart") %in% flags)) {
     if(! exists("slurmConfig")) {
       slurmConfig <- choose_slurmConfig(flags = flags)
     }
-    if ("--quick" %in% flags) slurmConfig <- combine_slurmConfig(slurmConfig, "--time=60")
+    if ("--quick" %in% flags && ! slurmConfig == "direct") slurmConfig <- combine_slurmConfig(slurmConfig, "--time=60")
     message()
     for (outputdir in outputdirs) {
       message("Restarting ", outputdir)

tests/testthat/test_01-readCheckScenarioConfig.R

@@ -16,14 +16,16 @@ for (csvfile in csvfiles) {
 }
 test_that("readCheckScenarioConfig fails on error-loaden config", {
   csvfile <- tempfile(pattern = "scenario_config_a", fileext = ".csv")
-  writeLines(c(";start;copyConfigFrom;c_budgetCO2;path_gdx;path_gdx_carbonprice",
-               "abc.loremipsumloremipsum@lorem&ipsumloremipsumloremipsumloremipsumloremipsumloremipsum_;0;;33;;",
-               "PBS;1;glob;29; whitespacebefore;whitespaceafter ",
-               "glob;0;missing_copyConfigFrom;33; ;nobreakspace	tab",
-               "PBScopy;0;PBS;;;mustbedifferenttoPBS"),
+  writeLines(c(";start;copyConfigFrom;c_budgetCO2;path_gdx;path_gdx_carbonprice;carbonprice;path_gdx_bau;path_gdx_ref",
+               "abc.loremipsumloremipsum@lorem&ipsumloremipsumloremipsumloremipsumloremipsumloremipsum_;0;;33;;;;;",
+               "PBS;1;glob;29; whitespacebefore;whitespaceafter ;;;whatever",
+               "glob;0;missing_copyConfigFrom;33; ;nobreakspace	tab;;;",
+               "PBScopy;0;PBS;;;mustbedifferenttoPBS;;;",
+               "NA;1;;;;;notNDC_but_has_path_gdx_bau;PBS;",
+               "NDC_but_bau_missing;1;;;;;NDC;;"),
              con = csvfile, sep = "\n")
-  w <- capture_warnings(scenConf <- readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE))
-  expect_match(w, "11 errors found", all = FALSE, fixed = TRUE)
+  w <- capture_warnings(m <- capture_messages(scenConf <- readCheckScenarioConfig(csvfile, remindPath = "../../", testmode = TRUE)))
+  expect_match(w, "13 errors found", all = FALSE, fixed = TRUE)
   expect_match(w, "These titles are too long", all = FALSE, fixed = TRUE)
   expect_match(w, "These titles may be confused with regions", all = FALSE, fixed = TRUE)
   expect_match(w, "These titles contain illegal characters", all = FALSE, fixed = TRUE)
@@ -32,6 +34,9 @@ test_that("readCheckScenarioConfig fails on error-loaden config", {
   expect_match(w, "contain whitespaces", all = FALSE, fixed = TRUE)
   expect_match(w, "scenario names indicated in copyConfigFrom column were not found", all = FALSE, fixed = TRUE)
   expect_match(w, "specify in copyConfigFrom column a scenario name defined below in the file", all = FALSE, fixed = TRUE)
+  expect_match(w, "which requires a reference gdx", all = FALSE, fixed = TRUE)
+  expect_match(m, "no column path_gdx_refpolicycost for policy cost comparison found, using path_gdx_ref instead", all = FALSE, fixed = TRUE)
+  expect_match(m, "In 1 scenarios, neither 'carbonprice'", all = FALSE, fixed = TRUE)
   copiedFromPBS <- c("c_budgetCO2", "path_gdx", "path_gdx_ref")
   expect_identical(unlist(scenConf["PBS", copiedFromPBS]),
                    unlist(scenConf["PBScopy", copiedFromPBS]))

tests/testthat/test_01-start.R

@@ -22,6 +22,7 @@ test_that("start.R --test startgroup=AMT titletag=AMT config/scenario_config.csv
                            c("start.R", "--test", "slurmConfig=16", "startgroup=AMT", "titletag=TESTTHAT", "config/scenario_config.csv"))
     printIfFailed(output)
     expectSuccessStatus(output)
+    expect_false(any(grepl("Waiting for.* NA( |$)", output)))
     },
     getLine = function() stop("getLine should not called."),
     .package = "gms"
@@ -48,6 +49,7 @@ test_that("start.R --test succeeds on all configs", {
                              c("start.R", "--test", "slurmConfig=16", "startgroup=*", "titletag=TESTTHAT", csvfile))
         printIfFailed(output)
         expectSuccessStatus(output)
+        expect_false(any(grepl("Waiting for.* NA( |$)", output)))
       })
     },
     getLine = function() stop("getLine should not called."),