Irace callback feature

(cherry picked from commit bec759c)

Author: rmartinsanta

NEWS.md

@@ -1,19 +1,24 @@
 **If you are viewing this file on CRAN, please check [latest news on the irace website](https://mlopez-ibanez.github.io/irace/news/index.html) where the formatting is also better.**
 
-# irace 4.4.9000
 
+# irace 4.5
+
+## New features and improvements
+
+* #98 irace callback: Add an optional `iterationCallback` scenario function that is invoked after each completed irace iteration.
+  This allows irace users to monitor tuning progress or collect intermediate results without modifying irace internals. Check the user guide for more details and an example implementation.
 
 # irace 4.4.4
 
- * The package contains an example of using `irace` to tune algorithms
-   implemented in Julia language.
+* The package contains an example of using `irace` to tune algorithms
+  implemented in Julia language.
 
- * Fix #94: `sample_model.ParamOrd()` now samples uniformly when the model
-   mean is `NA`, avoiding `NA` values for active ordinal parameters.
-   Reported by @Saethox.
+* Fix #94: `sample_model.ParamOrd()` now samples uniformly when the model
+  mean is `NA`, avoiding `NA` values for active ordinal parameters.
+  Reported by @Saethox.
 
- * Fix #97: `psRace()` errors when the post-selection race has nothing left to
-   run. Fixed by @ivchicano.
+* Fix #97: `psRace()` errors when the post-selection race has nothing left to
+  run. Fixed by @ivchicano.
 
 # irace 4.4.3
 

R/irace-options.R

@@ -16,19 +16,20 @@
 "minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime",
 "parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel",
 "seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances",
-"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest",
-"cappingType", "boundType", "boundMax", "boundDigits", "boundPar",
-"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration",
-"minNbSurvival", "nbConfigurations", "mu", "confidence"), type = c("x",
-"x", "x", "x", "p", "p", "p", "p", "x", "x", "p", "p", "p", "s",
-"p", "p", "b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p",
-"p", "s", "i", "i", "x", "x", "p", "b", "i", "i", "i", "r", "r",
-"i", "b", "b", "s", "b", "i", "i", "b", "r", "b", "i", "i", "x",
-"b", "b", "s", "s", "i", "i", "i", "b", "b", "b", "i", "i", "i",
-"i", "i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", "",
-"-p", "", "", "", "-l", "", "", "", "", "", "", "", "", "", "",
+"elitistLimit", "repairConfiguration", "iterationCallback", "capping",
+"cappingAfterFirstTest", "cappingType", "boundType", "boundMax",
+"boundDigits", "boundPar", "boundAsTimeout", "postselection",
+"aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival",
+"nbConfigurations", "mu", "confidence"), type = c("x", "x", "x",
+"x", "p", "p", "p", "p", "x", "x", "p", "p", "p", "s", "p", "p",
+"b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p", "p", "s",
+"i", "i", "x", "x", "p", "b", "i", "i", "i", "r", "r", "i", "b",
+"b", "s", "b", "i", "i", "b", "r", "b", "i", "i", "x", "x", "b",
+"b", "s", "s", "i", "i", "i", "b", "b", "b", "i", "i", "i", "i",
+"i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", "", "-p",
+"", "", "", "-l", "", "", "", "", "", "", "", "", "", "", "",
 "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
-"", "", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "",
+"", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "", "",
 "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""
 ), long = c("--help", "--version", "--check", "--init", "--only-test",
 "--scenario", "--exec-dir", "--parameter-file", "", "", "--configurations-file",
@@ -43,7 +44,7 @@
 "--parallel", "--load-balancing", "--mpi", "--batchmode", "--quiet",
 "--debug-level", "--seed", "--soft-restart", "--soft-restart-threshold",
 "--elitist", "--elitist-new-instances", "--elitist-limit", "",
-"--capping", "--capping-after-first-test", "--capping-type",
+"", "--capping", "--capping-after-first-test", "--capping-type",
 "--bound-type", "--bound-max", "--bound-digits", "--bound-par",
 "--bound-as-timeout", "--postselection", "--aclib", "--iterations",
 "--experiments-per-iteration", "--min-survival", "--num-configurations",
@@ -53,13 +54,13 @@
 "", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}",
 "0", "0", "", "", "", "0", "0", NA, "0", "0.05", "0.01", "0",
 "1", "0", "0", "0", "0", NA, "1", "1e-04", "1", "1", "2", "",
-NA, "0", "median", "candidate", "0", "0", "1", "1", "1", "0",
-"0", "0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA, NA,
+"", NA, "0", "median", "candidate", "0", "0", "1", "1", "1",
+"0", "0", "0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA,
 NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
-NA, NA, "F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA,
+NA, NA, NA, "F-test,t-test,t-test-holm,t-test-bonferroni", NA,
 NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,
-NA, NA, "sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA, NA,
-NA, NA, NA, NA, NA, NA, "median,mean,worst,best", "instance,candidate",
+NA, NA, NA, "sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA,
+NA, NA, NA, NA, NA, NA, NA, NA, "median,mean,worst,best", "instance,candidate",
 NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help.",
 "Show irace package version.", "Check scenario.", "Initialize the working directory with template config files.",
 "Only test the configurations given in the file passed as argument.",
@@ -103,6 +104,7 @@ NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help
 "Number of instances added to the execution list before previous instances in elitist irace.",
 "In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit.",
 "User-defined R function that takes a configuration generated by irace and repairs it.",
+"Optional R function called after each irace iteration with the iteration number, ordered elite configurations, and progress information.",
 "Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and  `boundMax > 0`.",
 "If set to 1, elimination due to capping only happens after `firstTest` instances are seen.",
 "Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best.",
@@ -130,10 +132,11 @@ NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help
 "maxTime", "budgetEstimation", "minMeasurableTime", "parallel",
 "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed",
 "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances",
-"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest",
-"cappingType", "boundType", "boundMax", "boundDigits", "boundPar",
-"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration",
-"minNbSurvival", "nbConfigurations", "mu", "confidence"), class = "data.frame")
+"elitistLimit", "repairConfiguration", "iterationCallback", "capping",
+"cappingAfterFirstTest", "cappingType", "boundType", "boundMax",
+"boundDigits", "boundPar", "boundAsTimeout", "postselection",
+"aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival",
+"nbConfigurations", "mu", "confidence"), class = "data.frame")
 .irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations",
 "configurationsFile", "logFile", "recoveryFile", "instances",
 "trainInstancesDir", "trainInstancesFile", "sampleInstances",
@@ -145,10 +148,11 @@ NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help
 "minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime",
 "parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel",
 "seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances",
-"elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest",
-"cappingType", "boundType", "boundMax", "boundDigits", "boundPar",
-"boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration",
-"minNbSurvival", "nbConfigurations", "mu", "confidence")
+"elitistLimit", "repairConfiguration", "iterationCallback", "capping",
+"cappingAfterFirstTest", "cappingType", "boundType", "boundMax",
+"boundDigits", "boundPar", "boundAsTimeout", "postselection",
+"aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival",
+"nbConfigurations", "mu", "confidence")
 ## FIXME: If these values are special perhaps they should be saved in $state ?
 .irace.params.recover <- c("instances", "seed", "testInstances",
                            # We need this because this data may mutate

R/irace.R

@@ -1213,6 +1213,27 @@ irace_run <- function(scenario)
         race_state$print_mem_used()
       }
     }
+    if (!is.null(scenario$iterationCallback)) {
+      progress <- list(
+        nbIterations = nbIterations,
+        maxExperiments = scenario$maxExperiments,
+        experimentsUsed = experimentsUsed,
+        remainingBudget = remainingBudget,
+        remainingBudgetEstimated = scenario$maxTime > 0L,
+        currentBudget = currentBudget,
+        currentBudgetUsed = raceResults$experimentsUsed,
+        maxTime = scenario$maxTime,
+        timeUsed = timeUsed,
+        remainingTime = if (scenario$maxTime > 0L)
+                          scenario$maxTime - timeUsed
+                        else NA_real_,
+        boundEstimate = boundEstimate)
+      withr::with_preserve_seed(
+        scenario$iterationCallback(
+          iteration = indexIteration,
+          elites = data.table::copy(elite_configurations),
+          progress = progress))
+    }
     indexIteration <- indexIteration + 1L
   } # end of repeat
 }

R/scenario.R

@@ -316,6 +316,13 @@ checkScenario <- function(scenario = defaultScenario())
   else
     irace_error("'repairConfiguration' must be a function")
 
+  if (is.null.or.empty(scenario$iterationCallback))
+    scenario$iterationCallback <- NULL
+  else if (is.function.name(scenario$iterationCallback))
+    scenario$iterationCallback <- get.function(scenario$iterationCallback)
+  else
+    irace_error("'iterationCallback' must be a function")
+
   if (is.na(scenario$capping))
     scenario$capping <- (scenario$elitist && scenario$maxTime > 0 &&
                            !is.na(scenario$boundMax) && scenario$boundMax > 0)
@@ -565,6 +572,7 @@ printScenario <- function(scenario)
 #'      \item{`debugLevel`}{Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. (Default: `0`)}
 #'      \item{`seed`}{Seed of the random number generator (by default, generate a random seed). (Default: `NA`)}
 #'      \item{`repairConfiguration`}{User-defined R function that takes a configuration generated by irace and repairs it. (Default: `""`)}
+#'      \item{`iterationCallback`}{Optional R function called after each irace iteration with the iteration number, ordered elite configurations, and progress information. (Default: `""`)}
 #'      \item{`postselection`}{Perform a postselection race after the execution of irace to consume all remaining budget (requires elitist racing). Value 0 disables the postselection race. (Default: `1`)}
 #'      \item{`aclib`}{Enable/disable AClib mode. This option enables compatibility with GenericWrapper4AC as targetRunner script. (Default: `0`)}
 #'    }

inst/templates/scenario.txt.tmpl

@@ -188,6 +188,10 @@
 ## repairs it.
 # repairConfiguration = ""
 
+## Optional R function called after each irace iteration with the iteration
+## number, ordered elite configurations, and progress information.
+# iterationCallback = ""
+
 ## Enable the use of adaptive capping, a technique designed for minimizing the
 ## computation time of configurations. Capping is enabled by default if
 ## `elitist` is active, `maxTime > 0` and `boundMax > 0`.

man/defaultScenario.Rd

@@ -521,6 +521,16 @@
     "vignettes": "User-defined \\aR function that takes a configuration generated by \\irace and repairs it. See \\autoref{sec:repairconf} for details.",
     "section": "General options"
   },
+  {
+    "name": "iterationCallback",
+    "type": "x",
+    "short": "",
+    "long": "",
+    "default": "",
+    "description": "Optional R function called after each irace iteration with the iteration number, ordered elite configurations, and progress information.",
+    "vignettes": "Optional \\aR function called after each \\irace iteration with the iteration number, ordered elite configurations, and progress information. See \\autoref{sec:iteration callback} for details.",
+    "section": "General options"
+  },
   {
     "name": "capping",
     "type": "b",

scripts/irace_options.json

@@ -0,0 +1,108 @@
+callback_scenario <- function(log_file)
+{
+  parameters <- readParameters(text = 'x "" i (1, 10)')
+  list(
+    targetRunner = function(experiment, scenario) {
+      list(cost = abs(experiment$configuration[["x"]] - experiment$instance))
+    },
+    instances = seq_len(10L),
+    maxExperiments = 100L,
+    seed = 1234567L,
+    quiet = TRUE,
+    postselection = FALSE,
+    logFile = log_file,
+    parameters = parameters
+  )
+}
+
+test_that("iterationCallback receives named arguments after every iteration", {
+  log_file <- withr::local_tempfile(fileext = ".Rdata")
+  snapshots <- list()
+  # The arguments are deliberately reordered to verify that irace calls the
+  # callback by name. The progress argument is collected through ... to ensure
+  # callbacks remain compatible when irace adds named arguments.
+  callback <- function(elites, iteration, ...) {
+    dots <- list(...)
+    expect_named(dots, "progress")
+    snapshots[[iteration]] <<- list(
+      elites = data.table::copy(elites),
+      progress = dots$progress)
+    runif(1L)
+    data.table::set(elites, i = 1L, j = ".ID.", value = -1L)
+  }
+
+  scenario <- callback_scenario(log_file)
+  scenario$iterationCallback <- callback
+  with_callback <- irace(scenario)
+  results <- read_logfile(log_file)
+
+  expect_length(snapshots, length(results$allElites))
+  for (iteration in seq_along(snapshots)) {
+    progress <- snapshots[[iteration]]$progress
+    expect_identical(
+      snapshots[[iteration]]$elites[[".ID."]],
+      results$allElites[[iteration]])
+    expect_named(progress, c(
+      "nbIterations", "maxExperiments", "experimentsUsed",
+      "remainingBudget", "remainingBudgetEstimated", "currentBudget",
+      "currentBudgetUsed", "maxTime", "timeUsed", "remainingTime",
+      "boundEstimate"))
+    expect_gte(progress$nbIterations, iteration)
+    expect_identical(progress$maxExperiments, 100L)
+    expect_identical(progress$experimentsUsed + progress$remainingBudget,
+                     progress$maxExperiments)
+    expect_false(progress$remainingBudgetEstimated)
+    expect_gte(progress$currentBudget, progress$currentBudgetUsed)
+    expect_gt(progress$currentBudgetUsed, 0L)
+    expect_identical(progress$maxTime, 0L)
+    expect_identical(progress$timeUsed, 0)
+    expect_true(is.na(progress$remainingTime))
+    expect_true(is.na(progress$boundEstimate))
+  }
+  expect_true(all(with_callback[[".ID."]] > 0L))
+
+  without_callback <- irace(
+    callback_scenario(withr::local_tempfile(fileext = ".Rdata")))
+  expect_identical(with_callback, without_callback)
+})
+
+test_that("iterationCallback reports time-budget progress", {
+  snapshots <- list()
+  callback <- function(iteration, elites, progress, ...) {
+    snapshots[[iteration]] <<- progress
+  }
+  scenario <- callback_scenario("")
+  scenario$maxExperiments <- 0L
+  scenario$maxTime <- 100
+  scenario$targetRunner <- function(experiment, scenario) {
+    list(
+      cost = abs(experiment$configuration[["x"]] - experiment$instance),
+      time = 1)
+  }
+  scenario$iterationCallback <- callback
+
+  irace(scenario)
+
+  expect_gt(length(snapshots), 0L)
+  for (progress in snapshots) {
+    expect_identical(progress$maxExperiments, 0L)
+    expect_identical(progress$maxTime, 100L)
+    expect_true(progress$remainingBudgetEstimated)
+    expect_gt(progress$experimentsUsed, 0L)
+    expect_gte(progress$currentBudget, progress$currentBudgetUsed)
+    expect_equal(progress$remainingTime,
+                 progress$maxTime - progress$timeUsed)
+    expect_equal(progress$boundEstimate,
+                 progress$timeUsed / progress$experimentsUsed)
+  }
+})
+
+test_that("iterationCallback must be NULL or a function", {
+  scenario <- callback_scenario("")
+  scenario$iterationCallback <- "invalid"
+  expect_error(
+    irace(scenario),
+    "'iterationCallback' must be a function",
+    fixed = TRUE
+  )
+})

tests/testthat/test-iteration-callback.R

@@ -2283,6 +2283,57 @@ the ACVIZ software package~\cite{DesRitLopPer2021acviz}, which is available at \
 
 \section{Advanced topics}
 
+\subsection{Monitoring iterations with a callback}
+\label{sec:iteration callback}
+
+The scenario option \parameter{iterationCallback} may be set to an \aR
+function that is called after every completed \irace iteration. For example,
+the following callback reports the consumed budget and the identifier of the
+best configuration found in each iteration:
+
+<<iterationCallback,prompt=FALSE,eval=FALSE>>=
+report_progress <- function(iteration, elites, progress, ...)
+{
+  if (progress$maxTime > 0) {
+    used <- progress$timeUsed
+    total <- progress$maxTime
+    budget_type <- "time"
+  } else {
+    used <- progress$experimentsUsed
+    total <- progress$maxExperiments
+    budget_type <- "experiments"
+  }
+  message("Iteration ", iteration, "/", progress$nbIterations,
+          ": ", used, "/", total, " ", budget_type,
+          " used; best configuration: ", elites[[".ID."]][1L])
+}
+scenario$iterationCallback <- report_progress
+irace(scenario)
+@
+
+The callback is called with the named arguments \code{iteration},
+\code{elites}, and \code{progress}. User-defined callbacks should use these
+exact argument names and take \code{...} as their last argument so they remain
+compatible if future versions of \irace provide more information. The
+\code{elites} data frame is ordered from best to worst and is copied before it
+is passed to the callback.
+
+The \code{progress} list contains \code{nbIterations},
+\code{maxExperiments}, \code{experimentsUsed}, \code{remainingBudget},
+\code{remainingBudgetEstimated}, \code{currentBudget},
+\code{currentBudgetUsed}, \code{maxTime}, \code{timeUsed},
+\code{remainingTime}, and \code{boundEstimate}. The budgets
+\code{remainingBudget}, \code{currentBudget}, and \code{currentBudgetUsed}
+are numbers of target-algorithm executions. When \parameter{maxTime} is used,
+\code{remainingBudget} is estimated from \code{boundEstimate}, and
+\code{remainingBudgetEstimated} is \code{TRUE}. In that mode,
+\code{remainingTime} uses the same unit as \parameter{maxTime}; otherwise,
+\code{remainingTime} and \code{boundEstimate} are \code{NA}.
+
+The callback return value is ignored. Errors raised by the callback stop
+\irace. Random-number generation within the callback does not affect the
+random-number sequence of the tuning run.
+
 \subsection{Tuning budget}\label{sec:budget}
 
 Before setting the budget for a run of \irace, please consider the number of