Prepare 0.9-2 lifecycle, provenance, and smoothness release

.gitignore

@@ -54,6 +54,7 @@ local/
 tmp/
 .cache/
 /inst/dev/
+/postprocess_roi_e2e_*/
 
 .DS_Store
 .vscode/

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: BrainGnomes
 Title: HPC Workflows for Functional MRI Preprocessing and Analysis
-Version: 0.9-1
-Date: 2026-08-27
+Version: 0.9-2
+Date: 2026-08-30
 Authors@R: 
     person("Michael", "Hallquist", email="michael.hallquist@gmail.com", role = c("aut", "cre"))
 Description: Configures, submits, and monitors reproducible functional magnetic

NAMESPACE

@@ -1,42 +1,64 @@
 # Generated by roxygen2: do not edit by hand
 
+S3method(print,bg_project_diagnosis)
+S3method(print,bg_project_doctor)
+S3method(print,bg_project_plan)
+S3method(print,bg_project_run)
+S3method(print,bg_project_validation)
+S3method(print,bg_run_provenance)
 S3method(summary,bg_project_cfg)
 S3method(summary,bg_status_df)
 export(add_tracked_job_parent)
 export(automask)
 export(butterworth_filter_4d)
 export(calculate_motion_outliers)
+export(cancel_project_run)
 export(cluster_job_submit)
 export(construct_bids_filename)
 export(construct_bids_regex)
 export(derive_reference_core)
 export(diagnose_pipeline)
+export(diagnose_project)
+export(doctor)
+export(doctor_project)
 export(edit_project)
 export(extract_bids_info)
 export(extract_rois)
 export(filtfilt_cpp)
+export(find_run_logs)
 export(get_fmriprep_outputs)
 export(get_postproc_output_files)
+export(get_project_runs)
 export(get_project_status)
+export(get_run_jobs)
+export(get_run_provenance)
 export(get_subject_status)
 export(get_tracked_job_status)
 export(getline)
 export(image_quantile)
+export(initialize_project)
 export(insert_tracked_job)
 export(lmfit_residuals_4d)
 export(load_project)
 export(natural_spline_4d)
 export(natural_spline_interp)
 export(parse_cli_args)
+export(plan_project)
 export(postprocess_subject)
+export(read_project_plan)
 export(remove_nifti_volumes)
 export(resample_template_to_img)
+export(retry_project_run)
 export(run_bids_validation)
 export(run_fsl_command)
 export(run_project)
 export(setup_project)
+export(submit_project_plan)
 export(update_tracked_job_status)
+export(validate_project_config)
 export(wait_for_job)
+export(write_project_config)
+export(write_project_plan)
 import(RNifti)
 importFrom(DBI,dbBegin)
 importFrom(DBI,dbCommit)

NEWS.md

@@ -1,3 +1,59 @@
+# BrainGnomes 0.9-2
+
+Released 2026-08-30
+
+* Extend the established `setup_project()` -> `run_project()` workflow across the
+  complete R and command-line lifecycle without adding required setup steps.
+  Optional inspection tooling now provides non-mutating configuration validation,
+  `doctor()` preflight checks, and serializable execution plans; direct runs
+  resolve the same execution model exposed by plans. Run handles, tracked-run
+  views, log discovery, non-interactive diagnosis, failed-job retry planning, and
+  guarded scheduler cancellation support observation and recovery. Flywheel
+  controller snapshots are now run-specific so concurrent submissions cannot
+  overwrite one another.
+* Record complete provenance before each run begins submission. Every run now
+  retains its resolved request and subject scope, exact configuration, planned
+  resources and dependencies, software/R/host/scheduler identity, and content
+  fingerprints for selected containers and other execution-driving files.
+  `get_run_provenance()` and the `BrainGnomes provenance` command expose the
+  record together with current job-tracking rows; plan and retry origins are
+  retained explicitly.
+* Expand applied-user recovery documentation around the established workflow.
+  The Quickstart and diagnosis guide now show how to select one run, inspect its
+  failed jobs, logs, and provenance, preview a retry, submit it as a separate
+  new run, decide whether to include downstream blocked work, and safely preview
+  cancellation. Function and CLI help use the same plain-language behavior and
+  safety guidance.
+* Recalibrate masked-SUSAN validation on real fMRIPrep BOLD data for the
+  distinct no-input-mask, fMRIPrep-mask, and TemplateFlow-mask conditions.
+  Validation now enforces the selected detrending-plus-MAD estimator, uses up
+  to 96 timepoints deterministically distributed over the complete run (or all
+  timepoints in shorter runs), reads only those volumes with RNifti, and cannot
+  pass by extrapolating across input-mask, kernel-size, voxel-size, or sampling
+  support. Calibration retains the full-run SUSAN threshold, temporal mean, and
+  extents while estimating smoothness from the selected timepoints.
+* Validate the promoted 3--8 mm masked-SUSAN models on independent fMRIPrep
+  25.2.5 derivatives and a 96-volume postprocessing E2E fixture drawn across
+  the complete run.
+  The 10 mm stress kernel remains outside the supported calibration range.
+* Allow Slurm and PBS fsaverage setup to copy with GNU `cp` when newer fMRIPrep
+  containers do not provide `rsync`, while retaining the existing `rsync` path
+  when present.
+* Strengthen postprocessing validation so masking is replayed exactly,
+  interpolation preserves retained volumes and matches sampled natural-spline
+  values, removed volumes match the censor vector in order, and AROMA/confound
+  regression samples are deterministic, pre-selected, and spatially balanced
+  across image resolutions.
+* Make temporal-filter validation deterministic and pre-selected, require
+  finite per-voxel stopband and passband evidence, verify that no-noise-IC
+  AROMA output is actually unchanged, reject wholly invalid AROMA component
+  requests, and fail every image validator when spatial NIfTI grid metadata
+  change unexpectedly.
+* Route postprocessing checks through a common validation runner: validator
+  errors now obey the configured continue/stop policy, reused intermediates are
+  checked, structured results are retained in a JSON audit beside the subject
+  log, and final images remain staged until last-step validation completes.
+
 # BrainGnomes 0.9-1
 
 Released 2026-08-27

R/cli_functions.R

@@ -202,7 +202,7 @@ parse_cli_args <- function(args, sep = "/", type_values = TRUE) {
 # Translate the public command-line options into run_project() arguments.
 # Keeping this adapter in the package makes the installed CLI behavior directly
 # testable without submitting scheduler jobs from a subprocess.
-.run_project_cli <- function(input, cli_args = list()) {
+run_project_cli <- function(input, cli_args = list()) {
   checkmate::assert_string(input)
   checkmate::assert_list(cli_args)
 
@@ -214,6 +214,10 @@ parse_cli_args <- function(args, sep = "/", type_values = TRUE) {
   if (is.null(dry_run_opt)) dry_run_opt <- cli_args$dry_run
 
   scfg <- load_project(input)
+  attr(scfg, "provenance_context") <- list(
+    interface = "cli",
+    input = normalizePath(input, winslash = "/", mustWork = FALSE)
+  )
   run_project(
     scfg,
     steps = cli_args$steps,
@@ -222,7 +226,8 @@ parse_cli_args <- function(args, sep = "/", type_values = TRUE) {
     extract_streams = cli_args$extract_streams,
     debug = cli_flag(cli_args$debug),
     force = cli_flag(cli_args$force),
-    dry_run = cli_flag(dry_run_opt)
+    dry_run = cli_flag(dry_run_opt),
+    log_level = if (is.null(cli_args$log_level)) "INFO" else cli_args$log_level
   )
 }
 

R/diagnose_pipeline.R

@@ -1,6 +1,16 @@
-#' Function for diagnosing errors in a run of the pipeline
+#' Interactively investigate jobs and logs from pipeline runs
 #'
-#' @param input A character path to an scfg object or an scfg object itself.
+#' Opens the established guided diagnosis browser. You can start with one
+#' subject across runs or select one run, follow its job relationships, and
+#' inspect output or error logs. Diagnosis does not submit jobs or change the
+#' project. Use [diagnose_project()] instead when a script or report needs a
+#' prompt-free summary.
+#'
+#' @param input A project configuration object or project directory.
+#' @return Depending on the selected action, the chosen run's job tree, log
+#'   contents, or invisibly `NULL`.
+#' @seealso [get_project_runs()] to find run IDs, [diagnose_project()] for a
+#'   prompt-free summary, and [retry_project_run()] after correcting a failure.
 #'
 #' @importFrom cli cli_abort cli_warn cli_inform cli_alert no qty
 #' @export
@@ -589,7 +599,7 @@ print_subject_summary_tree <- function(subject_jobs_df, subject_id) {
     return(invisible(NULL))
   }
 
-  status_priority <- c("COMPLETED" = 5, "STARTED" = 4, "QUEUED" = 3, "FAILED_BY_EXT" = 2, "FAILED" = 1)
+  status_priority <- c("COMPLETED" = 6, "STARTED" = 5, "QUEUED" = 4, "CANCELLED" = 3, "FAILED_BY_EXT" = 2, "FAILED" = 1)
 
   # Build trees to get hierarchy structure
   trees <- tracking_df_to_tree(subject_jobs_df)
@@ -891,6 +901,7 @@ get_status_symbol <- Vectorize(
            "STARTED" = cli::col_yellow("\u22ef"), # ellipsis
            "FAILED" = cli::col_red("\u2717"), # X mark
            "FAILED_BY_EXT" = cli::col_red("\u2717"), # X mark
+           "CANCELLED" = cli::col_red("\u25a0"), # stop square
            cli::col_yellow("?")) # unknown/unrecognized status
   },
   USE.NAMES = FALSE
@@ -910,6 +921,7 @@ get_status_color <- Vectorize(
            "STARTED" = cli::col_yellow(status),
            "FAILED" = cli::col_red(status),
            "FAILED_BY_EXT" = cli::col_red(status),
+           "CANCELLED" = cli::col_red(status),
            status)
   },
   USE.NAMES = FALSE

R/hpc_functions.R

@@ -366,13 +366,13 @@ wait_for_job <- function(job_ids, repolling_interval = 60, max_wait = 60 * 60 *
       } else {
         return(FALSE)
       }
-    } else if (all(status %in% c("failed", "complete"))) {
+    } else if (all(status %in% c("failed", "complete", "cancelled"))) {
       job_complete <- TRUE # drop out of this loop
       if (isFALSE(quiet)) {
         cat("All jobs have finished.\n")
       }
-      if (any(status == "failed")) {
-        cat("The following jobs(s) failed:", paste(job_ids[status == "failed"], collapse = ", "), "\n")
+      if (any(status %in% c("failed", "cancelled"))) {
+        cat("The following job(s) failed or were cancelled:", paste(job_ids[status %in% c("failed", "cancelled")], collapse = ", "), "\n")
         ret_code <- FALSE
       } else {
         ret_code <- TRUE

R/job_tracking_functions.R

@@ -339,8 +339,8 @@ update_tracked_job_status <- function(sqlite_db = NULL, job_id = NULL, status,
 
   checkmate::assert_string(status)
   status <- toupper(status)
-  checkmate::assert_subset(status, c("QUEUED", "STARTED", "FAILED", "COMPLETED", "FAILED_BY_EXT"))
-  if (cascade && status %in% c("QUEUED", "STARTED", "COMPLETED")) {
+  checkmate::assert_subset(status, c("QUEUED", "STARTED", "FAILED", "COMPLETED", "FAILED_BY_EXT", "CANCELLED"))
+  if (cascade && status %in% c("QUEUED", "STARTED", "COMPLETED", "CANCELLED")) {
     cascade <- FALSE
     warning("Only status FAILED or FAILED_BY_EXT can cascade in `update_tracked_job_status`")
   }
@@ -351,7 +351,8 @@ update_tracked_job_status <- function(sqlite_db = NULL, job_id = NULL, status,
                        STARTED = "time_started",
                        FAILED = "time_ended",
                        COMPLETED = "time_ended",
-                       FAILED_BY_EXT = "time_ended"
+                       FAILED_BY_EXT = "time_ended",
+                       CANCELLED = "time_ended"
   )
 
   rows_updated <- tryCatch({