From 33860abf99990f11e15570d37366a8fca05613c4 Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Wed, 15 Jul 2026 22:19:29 -0400 Subject: [PATCH 1/8] config(experiments): add AORC2F distributed/lumped + LSTM daily/hourly templates New train-and-test configs for the AORC2F dHBV Q' forecast stores (distributed+UH-routing and lumped variants) and the NH-LSTM daily/hourly stores, all using the 2026-07-12 capacity-boosted frozen disagg head (hourly-lstm omits it, since disagg + an hourly-native source is a config error). Also corrects docs/nh-qprime-store-contract.md's stale divide count for daily_dhbv2_merit_unit_catchments.ic (288,421 -> 197,088; the store was rebuilt 2026-07-10 after the doc was written) and adds the untracked daily_dhbv_aorc2f_merit_unit_catchments.ic store to the table, both verified via `ddrs import --dry-run`. --- .../aorc2f_distributed_frozen_chunk1.yaml | 96 ++++++++++++++++++ .../aorc2f_lumped_frozen_chunk1.yaml | 97 +++++++++++++++++++ .../experiments/lstm_daily_frozen_chunk1.yaml | 95 ++++++++++++++++++ config/experiments/lstm_hourly_native.yaml | 82 ++++++++++++++++ docs/nh-qprime-store-contract.md | 10 +- 5 files changed, 378 insertions(+), 2 deletions(-) create mode 100644 config/experiments/aorc2f_distributed_frozen_chunk1.yaml create mode 100644 config/experiments/aorc2f_lumped_frozen_chunk1.yaml create mode 100644 config/experiments/lstm_daily_frozen_chunk1.yaml create mode 100644 config/experiments/lstm_hourly_native.yaml diff --git a/config/experiments/aorc2f_distributed_frozen_chunk1.yaml b/config/experiments/aorc2f_distributed_frozen_chunk1.yaml new file mode 100644 index 0000000..550c8f8 --- /dev/null +++ b/config/experiments/aorc2f_distributed_frozen_chunk1.yaml @@ -0,0 +1,96 @@ +# aorc2f_distributed_frozen_chunk1.yaml — full CONUS routing train against the +# distributed+UH-routing dHBV AORC2F v2 Q' forecast (upstream checkpoint +# CONUS2717_AORC2F_v2 ep69 — a DDR/PyTorch model that produced this store's +# Q' data; ddrs only reads the resulting streamflow, it does not load that +# checkpoint). Same capacity-boosted frozen disagg head as +# kan_disagg_conus_frozen_chunk1.yaml (hidden 16, 2 KanLayers, grid 20, +# chunk_days=1, warm-started from capacity_chunk1.mpk, FROZEN). +# +# Both the KAN head and the router run on GPU (sparse_solver: cuda). +# Companion arm: aorc2f_lumped_frozen_chunk1.yaml (same settings, lumped Q'). + +mode: training +workflow: train-and-test +geodataset: merit +device: 0 +seed: 42 +np_seed: 42 + +data_sources: + attributes: /home/tbindas/projects/ddr/data/merit_global_attributes_v2.nc + conus_adjacency: /home/tbindas/projects/ddr/data/merit_conus_adjacency.zarr + gages_adjacency: /home/tbindas/projects/ddr/data/merit_gages_conus_adjacency.zarr + streamflow: /mnt/ssd1/data/icechunk/daily_dhbv_aorc2f_merit_unit_catchments.ic + observations: /mnt/ssd1/data/icechunk/usgs_daily_observations + gages: /home/tbindas/projects/ddr/references/gage_info/gages_3000.csv + aorc_precip: /mnt/ssd1/data/aorc/merit_unit_catchments.zarr + +experiment: + batch_size: 64 + start_time: 1981/10/01 + end_time: 1995/09/30 + epochs: 5 + rho: 90 + shuffle: true + warmup: 5 + learning_rate: + 1: 0.001 + 3: 0.0005 + grad_clip_max_norm: 1.0 + +kan_head: + hidden_size: 21 + num_hidden_layers: 2 + grid: 50 + k: 2 + input_var_names: + - SoilGrids1km_clay + - aridity + - meanelevation + - meanP + - NDVI + - meanslope + - log10_uparea + - SoilGrids1km_sand + - ETPOT_Hargr + - Porosity + learnable_parameters: + - n + - q_spatial + - p_spatial + # Capacity-boosted disagg head — architecture MUST match capacity_chunk1.mpk + # (pretrain_disagg_capacity --chunk-days 1: hidden 16, 2 layers, grid 20, k 3). + disaggregation: + hidden_size: 16 + num_hidden_layers: 2 + grid: 20 + k: 3 + boundary_blend: 0.0 + chunk_days: 1 + pretrained_checkpoint: /home/tbindas/projects/ddrs/output/disagg_pretrain/capacity_chunk1.mpk + freeze: true + +params: + parameter_ranges: + n: [0.015, 0.25] + q_spatial: [0.0, 1.0] + p_spatial: [1.0, 200.0] + attribute_minimums: + discharge: 1.0e-4 + slope: 1.0e-3 + velocity: 0.01 + depth: 0.01 + bottom_width: 0.01 + defaults: + p_spatial: 21.0 + log_space_parameters: + - p_spatial + sparse_solver: cuda + use_cuda_graphs: false + use_leakance: false + +testing: + start_time: 1995/10/01 + end_time: 2010/09/30 + batch_size: 15 + rho: null diff --git a/config/experiments/aorc2f_lumped_frozen_chunk1.yaml b/config/experiments/aorc2f_lumped_frozen_chunk1.yaml new file mode 100644 index 0000000..b852db2 --- /dev/null +++ b/config/experiments/aorc2f_lumped_frozen_chunk1.yaml @@ -0,0 +1,97 @@ +# aorc2f_lumped_frozen_chunk1.yaml — full CONUS routing train against the +# lumped (AORC) dHBV AORC2F Q' forecast (upstream checkpoint +# CONUS2717_AORC2F_LUMPED ep63 — a DDR/PyTorch model that produced this +# store's Q' data; ddrs only reads the resulting streamflow, it does not +# load that checkpoint). Same capacity-boosted frozen disagg head as +# kan_disagg_conus_frozen_chunk1.yaml (hidden 16, 2 KanLayers, grid 20, +# chunk_days=1, warm-started from capacity_chunk1.mpk, FROZEN). +# +# Both the KAN head and the router run on GPU (sparse_solver: cuda). +# Companion arm: aorc2f_distributed_frozen_chunk1.yaml (same settings, +# distributed+UH-routing Q'). + +mode: training +workflow: train-and-test +geodataset: merit +device: 0 +seed: 42 +np_seed: 42 + +data_sources: + attributes: /home/tbindas/projects/ddr/data/merit_global_attributes_v2.nc + conus_adjacency: /home/tbindas/projects/ddr/data/merit_conus_adjacency.zarr + gages_adjacency: /home/tbindas/projects/ddr/data/merit_gages_conus_adjacency.zarr + streamflow: /mnt/ssd1/data/icechunk/daily_dhbv2_merit_unit_catchments.ic + observations: /mnt/ssd1/data/icechunk/usgs_daily_observations + gages: /home/tbindas/projects/ddr/references/gage_info/gages_3000.csv + aorc_precip: /mnt/ssd1/data/aorc/merit_unit_catchments.zarr + +experiment: + batch_size: 64 + start_time: 1981/10/01 + end_time: 1995/09/30 + epochs: 5 + rho: 90 + shuffle: true + warmup: 5 + learning_rate: + 1: 0.001 + 3: 0.0005 + grad_clip_max_norm: 1.0 + +kan_head: + hidden_size: 21 + num_hidden_layers: 2 + grid: 50 + k: 2 + input_var_names: + - SoilGrids1km_clay + - aridity + - meanelevation + - meanP + - NDVI + - meanslope + - log10_uparea + - SoilGrids1km_sand + - ETPOT_Hargr + - Porosity + learnable_parameters: + - n + - q_spatial + - p_spatial + # Capacity-boosted disagg head — architecture MUST match capacity_chunk1.mpk + # (pretrain_disagg_capacity --chunk-days 1: hidden 16, 2 layers, grid 20, k 3). + disaggregation: + hidden_size: 16 + num_hidden_layers: 2 + grid: 20 + k: 3 + boundary_blend: 0.0 + chunk_days: 1 + pretrained_checkpoint: /home/tbindas/projects/ddrs/output/disagg_pretrain/capacity_chunk1.mpk + freeze: true + +params: + parameter_ranges: + n: [0.015, 0.25] + q_spatial: [0.0, 1.0] + p_spatial: [1.0, 200.0] + attribute_minimums: + discharge: 1.0e-4 + slope: 1.0e-3 + velocity: 0.01 + depth: 0.01 + bottom_width: 0.01 + defaults: + p_spatial: 21.0 + log_space_parameters: + - p_spatial + sparse_solver: cuda + use_cuda_graphs: false + use_leakance: false + +testing: + start_time: 1995/10/01 + end_time: 2010/09/30 + batch_size: 15 + rho: null diff --git a/config/experiments/lstm_daily_frozen_chunk1.yaml b/config/experiments/lstm_daily_frozen_chunk1.yaml new file mode 100644 index 0000000..8046d64 --- /dev/null +++ b/config/experiments/lstm_daily_frozen_chunk1.yaml @@ -0,0 +1,95 @@ +# lstm_daily_frozen_chunk1.yaml — full CONUS routing train against the daily +# CudaLSTM (NH) Q' forecast, updated to the capacity-boosted frozen disagg +# head (hidden 16, 2 KanLayers, grid 20, chunk_days=1, warm-started from +# capacity_chunk1.mpk, FROZEN) — supersedes the older +# equif_daily_lstm_disagg.yaml arm, which pre-dates the 2026-07-10/12 KAN +# disagg fixes and used a plain hidden_size:16 head with no pretraining. +# +# Both the KAN head and the router run on GPU (sparse_solver: cuda). +# Companion arm: lstm_hourly_native.yaml (hourly-native Q', no disagg head). + +mode: training +workflow: train-and-test +geodataset: merit +device: 0 +seed: 42 +np_seed: 42 + +data_sources: + attributes: /home/tbindas/projects/ddr/data/merit_global_attributes_v2.nc + conus_adjacency: /home/tbindas/projects/ddr/data/merit_conus_adjacency.zarr + gages_adjacency: /home/tbindas/projects/ddr/data/merit_gages_conus_adjacency.zarr + streamflow: /mnt/ssd1/data/icechunk/daily_lstm_merit_unit_catchments.ic + observations: /mnt/ssd1/data/icechunk/usgs_daily_observations + gages: /home/tbindas/projects/ddr/references/gage_info/gages_3000.csv + aorc_precip: /mnt/ssd1/data/aorc/merit_unit_catchments.zarr + +experiment: + batch_size: 64 + start_time: 1981/10/01 + end_time: 1995/09/30 + epochs: 5 + rho: 90 + shuffle: true + warmup: 5 + learning_rate: + 1: 0.001 + 3: 0.0005 + grad_clip_max_norm: 1.0 + +kan_head: + hidden_size: 21 + num_hidden_layers: 2 + grid: 50 + k: 2 + input_var_names: + - SoilGrids1km_clay + - aridity + - meanelevation + - meanP + - NDVI + - meanslope + - log10_uparea + - SoilGrids1km_sand + - ETPOT_Hargr + - Porosity + learnable_parameters: + - n + - q_spatial + - p_spatial + # Capacity-boosted disagg head — architecture MUST match capacity_chunk1.mpk + # (pretrain_disagg_capacity --chunk-days 1: hidden 16, 2 layers, grid 20, k 3). + disaggregation: + hidden_size: 16 + num_hidden_layers: 2 + grid: 20 + k: 3 + boundary_blend: 0.0 + chunk_days: 1 + pretrained_checkpoint: /home/tbindas/projects/ddrs/output/disagg_pretrain/capacity_chunk1.mpk + freeze: true + +params: + parameter_ranges: + n: [0.015, 0.25] + q_spatial: [0.0, 1.0] + p_spatial: [1.0, 200.0] + attribute_minimums: + discharge: 1.0e-4 + slope: 1.0e-3 + velocity: 0.01 + depth: 0.01 + bottom_width: 0.01 + defaults: + p_spatial: 21.0 + log_space_parameters: + - p_spatial + sparse_solver: cuda + use_cuda_graphs: false + use_leakance: false + +testing: + start_time: 1995/10/01 + end_time: 2010/09/30 + batch_size: 15 + rho: null diff --git a/config/experiments/lstm_hourly_native.yaml b/config/experiments/lstm_hourly_native.yaml new file mode 100644 index 0000000..07604ea --- /dev/null +++ b/config/experiments/lstm_hourly_native.yaml @@ -0,0 +1,82 @@ +# lstm_hourly_native.yaml — full CONUS routing train against the hourly-native +# MTS-LSTM (NH) Q' forecast. No disaggregation block — the store is already +# hourly-native, and kan_head.disaggregation + an hourly-native source is a +# hard config-load error (src/data/dataset.rs::validate_disagg_vs_resolution). +# +# Both the KAN head and the router run on GPU (sparse_solver: cuda). +# Companion arm: lstm_daily_frozen_chunk1.yaml (daily Q', frozen disagg head). + +mode: training +workflow: train-and-test +geodataset: merit +device: 0 +seed: 42 +np_seed: 42 + +data_sources: + attributes: /home/tbindas/projects/ddr/data/merit_global_attributes_v2.nc + conus_adjacency: /home/tbindas/projects/ddr/data/merit_conus_adjacency.zarr + gages_adjacency: /home/tbindas/projects/ddr/data/merit_gages_conus_adjacency.zarr + streamflow: /mnt/ssd1/data/icechunk/hourly_lstm_merit_unit_catchments.ic + observations: /mnt/ssd1/data/icechunk/usgs_daily_observations + gages: /home/tbindas/projects/ddr/references/gage_info/gages_3000.csv + aorc_precip: /mnt/ssd1/data/aorc/merit_unit_catchments.zarr + +experiment: + batch_size: 64 + start_time: 1981/10/01 + end_time: 1995/09/30 + epochs: 5 + rho: 90 + shuffle: true + warmup: 5 + learning_rate: + 1: 0.001 + 3: 0.0005 + grad_clip_max_norm: 1.0 + +kan_head: + hidden_size: 21 + num_hidden_layers: 2 + grid: 50 + k: 2 + input_var_names: + - SoilGrids1km_clay + - aridity + - meanelevation + - meanP + - NDVI + - meanslope + - log10_uparea + - SoilGrids1km_sand + - ETPOT_Hargr + - Porosity + learnable_parameters: + - n + - q_spatial + - p_spatial + +params: + parameter_ranges: + n: [0.015, 0.25] + q_spatial: [0.0, 1.0] + p_spatial: [1.0, 200.0] + attribute_minimums: + discharge: 1.0e-4 + slope: 1.0e-3 + velocity: 0.01 + depth: 0.01 + bottom_width: 0.01 + defaults: + p_spatial: 21.0 + log_space_parameters: + - p_spatial + sparse_solver: cuda + use_cuda_graphs: false + use_leakance: false + +testing: + start_time: 1995/10/01 + end_time: 2010/09/30 + batch_size: 15 + rho: null diff --git a/docs/nh-qprime-store-contract.md b/docs/nh-qprime-store-contract.md index 0db4e12..3a2351f 100644 --- a/docs/nh-qprime-store-contract.md +++ b/docs/nh-qprime-store-contract.md @@ -41,15 +41,21 @@ lives here. hourly-native — disaggregating an already-hourly signal is a config contradiction (`src/data/dataset.rs::validate_disagg_vs_resolution`). -## Conforming stores (2026-07-01) +## Conforming stores (2026-07-01, `daily_dhbv2`/`daily_dhbv_aorc2f` re-verified 2026-07-15 after a 2026-07-10 rebuild) | Store (`/mnt/ssd1/data/icechunk/`) | resolution | range | divides | |---|---|---|---| | `daily_lstm_merit_unit_catchments.ic` | daily | 1981-01-01 → 2020-12-30 | 288,421 | | `hourly_lstm_merit_unit_catchments.ic` | hourly | 1981-01-01 → 2020-12-31T23 | 197,088 | -| `daily_dhbv2_merit_unit_catchments.ic` | daily | 1980-01-01 → 2020-12-30 | 288,421 | +| `daily_dhbv2_merit_unit_catchments.ic` | daily | 1980-01-01 → 2020-12-31 | 197,088 | +| `daily_dhbv_aorc2f_merit_unit_catchments.ic` | daily | 1980-01-01 → 2020-12-31 | 197,088 | | `merit_dhbv2_UH_retrospective.ic` | daily | 1980-01-01 → 2020-12-31 | 197,088 | +`daily_dhbv2_merit_unit_catchments.ic` was rebuilt 2026-07-10 (previously +288,421 divides as of 2026-07-01); `ddrs import --dry-run` is the source of +truth for the current 197,088/197,084-fabric-coverage numbers, not this +table. + Note the hourly store starts **1981-01-01** (1980 was LSTM warmup): an experiment window reaching into 1980 hard-errors rather than clamping. From 8d432a8f581e58e7de1d2725b92813eadd65b7c3 Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Thu, 16 Jul 2026 00:58:27 -0400 Subject: [PATCH 2/8] fix(eval): detect silently-corrupted eval chunks instead of writing bad predictions Incident: a `--backend cuda` train-and-test run's Phase 2 (testing) hit a CUDA OOM inside a cubecl-cuda background worker thread ("can't allocate buffer of size: ..."). The panic never propagated to the caller -- the main thread kept looping through all 366 eval chunks, writing whatever half-written GPU buffer was left, instead of failing. Adds `DataError::CorruptedEvalChunk` and a `corrupted_chunk_reason` check (all-zero or non-finite chunk values) run after every eval chunk in `evaluate()`, which now takes a `checkpoint_path` for error context. Threads the new parameter through all four call sites (`cli/run.rs` Phase 2, `bin/eval.rs` both EvalParams arms, `bin/train_and_test.rs`, `tests/training_verification.rs`). Also adds a best-effort `system::free_gpu_gb()` warn-level log before Phase 2 begins (not gated -- cubecl's retained memory pool makes a point-in-time threshold check unreliable). 4 unit tests cover corrupted_chunk_reason (healthy/all-zero/nan/inf). --- src/bin/eval.rs | 21 ++++++++-- src/bin/train_and_test.rs | 1 + src/cli/run.rs | 10 +++++ src/cli/system.rs | 14 +++++++ src/data/error.rs | 13 +++++++ src/training/eval.rs | 70 +++++++++++++++++++++++++++++++++- tests/training_verification.rs | 3 +- 7 files changed, 127 insertions(+), 5 deletions(-) diff --git a/src/bin/eval.rs b/src/bin/eval.rs index 49f247e..7624b0c 100644 --- a/src/bin/eval.rs +++ b/src/bin/eval.rs @@ -117,7 +117,14 @@ fn run(cfg: Config, cli: Cli, device: I::Device) -> Result<(), Box(&cfg, &dataset, EvalParams::Frozen(&frozen), &device, cli.batch_size_days)? + evaluate::( + &cfg, + &dataset, + EvalParams::Frozen(&frozen), + &device, + cli.batch_size_days, + &cli.output, + )? } else { let head_section = cfg .kan_head @@ -125,8 +132,16 @@ fn run(cfg: Config, cli: Cli, device: I::Device) -> Result<(), Box = head_cfg.init::(&device); - let head = load_kan_head::(&head_base(cli.checkpoint.as_ref().unwrap()), head_template, &device)?; - evaluate::(&cfg, &dataset, EvalParams::KanHead(&head), &device, cli.batch_size_days)? + let checkpoint = cli.checkpoint.as_ref().unwrap(); + let head = load_kan_head::(&head_base(checkpoint), head_template, &device)?; + evaluate::( + &cfg, + &dataset, + EvalParams::KanHead(&head), + &device, + cli.batch_size_days, + checkpoint, + )? }; // Write the zarr. diff --git a/src/bin/train_and_test.rs b/src/bin/train_and_test.rs index 9a8a091..44daec2 100644 --- a/src/bin/train_and_test.rs +++ b/src/bin/train_and_test.rs @@ -166,6 +166,7 @@ fn main() -> Result<(), Box> { EvalParams::KanHead(&head), &device, cli.batch_size_days, + &latest_ckpt, )?; let phase2_elapsed = phase2_start.elapsed(); println!("Phase 2 complete in {:.2} min", phase2_elapsed.as_secs_f32() / 60.0); diff --git a/src/cli/run.rs b/src/cli/run.rs index db15bb2..ab9b3c0 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -417,6 +417,15 @@ where drop(train_dataset); // --- Phase 2: testing --- + // Testing's per-chunk buffers are far larger than a training + // minibatch's (whole-network dense state over batch_size_days), + // so free memory here is worth a log line even though it's not + // gated (see `system::free_gpu_gb` doc comment). + if !force_cpu { + if let Some(free_gb) = crate::cli::system::free_gpu_gb() { + eprintln!("Phase 2 (testing): free GPU memory {free_gb:.1} GB"); + } + } let phase2_start = Instant::now(); let mut test_cfg = Config::from_yaml_file_with_mode(&input.config_path, ConfigMode::Testing) .map_err(|e| CliError::Other(Box::new(e)))?; @@ -449,6 +458,7 @@ where EvalParams::KanHead(&head), &device, batch_size_days, + &latest, ) .map_err(|e| CliError::Other(Box::new(e)))?; diff --git a/src/cli/system.rs b/src/cli/system.rs index 1d8e639..8d2469b 100644 --- a/src/cli/system.rs +++ b/src/cli/system.rs @@ -98,6 +98,20 @@ pub fn record_smoke(probe: &mut SystemProbe, key: String, backend: &str) { }); } +/// Best-effort free-GPU-memory read for warn-level logging around a phase +/// transition (e.g. before Phase 2/testing, whose per-chunk buffers are far +/// larger than a training minibatch's). NOT a hard gate — cubecl's retained +/// memory pool means a moment-in-time free-memory read here is unreliable as +/// a threshold check (see the 2026-07-16 silent-eval-corruption incident, +/// `tests/eval_chunk_continuity.rs` and `DataError::CorruptedEvalChunk`, +/// which catch the actual failure mode instead). Returns `None` when no CUDA +/// context is current in this process. +pub fn free_gpu_gb() -> Option { + result::mem_get_info() + .ok() + .map(|(free, _total)| free as f32 / (1u64 << 30) as f32) +} + /// Result of [`ensure_system_ready`]. pub struct SystemReadiness { /// Post-write probe snapshot (matches what was persisted to diff --git a/src/data/error.rs b/src/data/error.rs index 70538a9..9b8ed02 100644 --- a/src/data/error.rs +++ b/src/data/error.rs @@ -60,6 +60,19 @@ pub enum DataError { #[source] source: csv::Error, }, + + #[error( + "eval chunk {chunk}/{total} looks corrupted ({message}); context path {path:?} — \ + likely a silent GPU worker-thread failure (e.g. a cubecl OOM in a background thread, \ + which logs-and-drops the task instead of propagating an error); retry with \ + --backend cpu or a smaller batch_size_days" + )] + CorruptedEvalChunk { + path: PathBuf, + chunk: usize, + total: usize, + message: String, + }, } pub type Result = std::result::Result; diff --git a/src/training/eval.rs b/src/training/eval.rs index b2e766f..966cb7f 100644 --- a/src/training/eval.rs +++ b/src/training/eval.rs @@ -6,13 +6,15 @@ //! threaded via `tensors.initial_state` (the final per-reach discharge column //! of each chunk is injected as the initial discharge of the next chunk). +use std::path::Path; + use burn::tensor::backend::Backend; use chrono::NaiveDate; use ndarray::{s, Array2}; use crate::config::Config; use crate::data::dataset::MeritGagesDataset; -use crate::data::error::Result; +use crate::data::error::{DataError, Result}; use crate::data::TestWindow; use crate::nn::kan_head::KanHead; use crate::training::{ @@ -47,12 +49,29 @@ pub struct EvalOutput { pub zeta_comids: Option>, } +/// Returns a diagnostic reason when a chunk's predictions are provably wrong +/// rather than merely low-skill: all exactly zero, or containing a +/// non-finite value. This is the fingerprint of the silent-corruption +/// failure mode where a cubecl-cuda background worker thread panics on an +/// OOM allocation and the panic never propagates to the caller — the main +/// thread just keeps looping with whatever half-written buffer it has. +fn corrupted_chunk_reason(pred: &Array2) -> Option { + if pred.iter().any(|v| !v.is_finite()) { + return Some("non-finite value(s) in predictions".to_string()); + } + if pred.iter().all(|&v| v == 0.0) { + return Some("all-zero predictions".to_string()); + } + None +} + pub fn evaluate( cfg: &Config, dataset: &MeritGagesDataset, params: EvalParams, device: &I::Device, batch_size_days: usize, + checkpoint_path: &Path, ) -> Result { let axis = dataset.time_axis().clone(); let n_days_total = axis.num_days; @@ -170,6 +189,14 @@ pub fn evaluate( } } let (pred_arr, final_col) = run_chunk(&win, prev_final_state.take())?; + if let Some(message) = corrupted_chunk_reason(&pred_arr) { + return Err(DataError::CorruptedEvalChunk { + path: checkpoint_path.to_path_buf(), + chunk: chunk_idx + 1, + total: n_chunks_total, + message, + }); + } prev_final_state = final_col; let h_start = day_offset * 24; let h_end = h_start + win.n_hourly(); @@ -285,3 +312,44 @@ pub fn evaluate( zeta_comids, }) } + +#[cfg(test)] +mod tests { + use super::corrupted_chunk_reason; + use ndarray::Array2; + + #[test] + fn healthy_chunk_is_not_corrupted() { + let pred = Array2::from_shape_vec((2, 3), vec![0.1, 1.0, 2.5, 0.0, 3.0, 4.0]).unwrap(); + assert!(corrupted_chunk_reason(&pred).is_none()); + } + + #[test] + fn all_zero_chunk_is_corrupted() { + let pred = Array2::::zeros((2, 3)); + assert_eq!( + corrupted_chunk_reason(&pred), + Some("all-zero predictions".to_string()) + ); + } + + #[test] + fn nan_chunk_is_corrupted() { + let mut pred = Array2::from_shape_vec((2, 2), vec![1.0, 2.0, 3.0, 4.0]).unwrap(); + pred[[1, 1]] = f32::NAN; + assert_eq!( + corrupted_chunk_reason(&pred), + Some("non-finite value(s) in predictions".to_string()) + ); + } + + #[test] + fn inf_chunk_is_corrupted() { + let mut pred = Array2::from_shape_vec((1, 2), vec![1.0, 2.0]).unwrap(); + pred[[0, 1]] = f32::INFINITY; + assert_eq!( + corrupted_chunk_reason(&pred), + Some("non-finite value(s) in predictions".to_string()) + ); + } +} diff --git a/tests/training_verification.rs b/tests/training_verification.rs index 55b7695..efea6d3 100644 --- a/tests/training_verification.rs +++ b/tests/training_verification.rs @@ -386,7 +386,8 @@ fn v4_test_period_matches_ddr_for_frozen_constant_params() { // Single batch covering the whole window — mirrors the dump script. let output = evaluate::(&cfg, &dataset, EvalParams::Frozen(&frozen), - &device, n_days_total).expect("evaluate"); + &device, n_days_total, std::path::Path::new("test-fixture")) + .expect("evaluate"); let pred_ddrs = &output.predictions_daily; eprintln!("V4: pred_ddrs shape {:?}", pred_ddrs.shape()); let ddrs_mean = pred_ddrs.mean().unwrap_or(0.0); From 724803078008ffb7afff57b74e81e830e115d76e Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Thu, 16 Jul 2026 09:14:05 -0400 Subject: [PATCH 3/8] fix(eval): strengthen corruption detection with a panic-hook flag A second, independent occurrence of the OOM incident from the previous commit showed the value-based corrupted_chunk_reason check is NOT sufficient on its own: the corrupted chunk happened to contain plausible-looking finite, non-zero values (stale GPU memory left over from a prior op) that slipped past the all-zero/non-finite heuristic while the cubecl-cuda worker thread panicked dozens of times underneath. Adds a process-global WORKER_PANICKED flag set by a custom panic hook (ensure_panic_hook_installed, wraps rather than replaces the default hook so panic messages still print) -- a reliable detector, since it fires on ANY background-thread panic regardless of what ends up in the output buffer. Checked after every chunk and once more after the post-loop tensor readback (tau-trim/downsample + zeta-mean readbacks also run on-device and were previously unchecked). 2 new unit tests (6 total in this module) confirm the hook fires across threads and the flag clears correctly between calls. --- src/training/eval.rs | 89 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/src/training/eval.rs b/src/training/eval.rs index 966cb7f..3b6c4a6 100644 --- a/src/training/eval.rs +++ b/src/training/eval.rs @@ -7,6 +7,8 @@ //! of each chunk is injected as the initial discharge of the next chunk). use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Once; use burn::tensor::backend::Backend; use chrono::NaiveDate; @@ -51,10 +53,10 @@ pub struct EvalOutput { /// Returns a diagnostic reason when a chunk's predictions are provably wrong /// rather than merely low-skill: all exactly zero, or containing a -/// non-finite value. This is the fingerprint of the silent-corruption -/// failure mode where a cubecl-cuda background worker thread panics on an -/// OOM allocation and the panic never propagates to the caller — the main -/// thread just keeps looping with whatever half-written buffer it has. +/// non-finite value. This is ONE fingerprint of the silent-corruption +/// failure mode below — kept as defense-in-depth alongside +/// [`worker_panic_detected`], since it doesn't depend on the panic hook +/// having fired. fn corrupted_chunk_reason(pred: &Array2) -> Option { if pred.iter().any(|v| !v.is_finite()) { return Some("non-finite value(s) in predictions".to_string()); @@ -65,6 +67,38 @@ fn corrupted_chunk_reason(pred: &Array2) -> Option { None } +/// Set by a global panic hook whenever ANY thread panics during an +/// `evaluate()` call. Primary detector for the silent-corruption failure +/// mode: a cubecl-cuda background worker thread (`DSD-0-0`) panics on a CUDA +/// OOM, but the panic never propagates to the caller and the main thread +/// keeps looping with whatever half-written buffer cubecl left behind. +/// `corrupted_chunk_reason`'s all-zero/non-finite check is NOT sufficient on +/// its own — confirmed 2026-07-16: a repeat of this exact incident produced +/// a chunk with plausible-looking finite, non-zero values (stale GPU memory +/// from a previous op) that slipped past it while the worker thread +/// panicked dozens of times underneath. +static WORKER_PANICKED: AtomicBool = AtomicBool::new(false); +static INSTALL_PANIC_HOOK: Once = Once::new(); + +/// Idempotent; wraps (does not replace) the default hook so panic messages +/// still print. Must be called before the eval chunk loop starts. +fn ensure_panic_hook_installed() { + INSTALL_PANIC_HOOK.call_once(|| { + let default_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + WORKER_PANICKED.store(true, Ordering::SeqCst); + default_hook(info); + })); + }); +} + +/// Clears any panic recorded before this call (e.g. from an unrelated prior +/// `evaluate()` invocation in the same process) and reports whether a panic +/// happened since. +fn take_worker_panicked() -> bool { + WORKER_PANICKED.swap(false, Ordering::SeqCst) +} + pub fn evaluate( cfg: &Config, dataset: &MeritGagesDataset, @@ -73,6 +107,9 @@ pub fn evaluate( batch_size_days: usize, checkpoint_path: &Path, ) -> Result { + ensure_panic_hook_installed(); + take_worker_panicked(); // clear any stale flag from an unrelated prior call + let axis = dataset.time_axis().clone(); let n_days_total = axis.num_days; assert!(batch_size_days > 0, "batch_size_days must be positive"); @@ -189,7 +226,9 @@ pub fn evaluate( } } let (pred_arr, final_col) = run_chunk(&win, prev_final_state.take())?; - if let Some(message) = corrupted_chunk_reason(&pred_arr) { + let panic_message = take_worker_panicked() + .then(|| "a background thread panicked during this chunk".to_string()); + if let Some(message) = panic_message.or_else(|| corrupted_chunk_reason(&pred_arr)) { return Err(DataError::CorruptedEvalChunk { path: checkpoint_path.to_path_buf(), chunk: chunk_idx + 1, @@ -298,6 +337,18 @@ pub fn evaluate( _ => (None, None, None, None, None, None), }; + // Final gate: the tau-trim/downsample and zeta-mean readbacks above also + // run on the device, after the last per-chunk check — a worker panic + // there would otherwise slip through as a clean `Ok`. + if take_worker_panicked() { + return Err(DataError::CorruptedEvalChunk { + path: checkpoint_path.to_path_buf(), + chunk: n_chunks_total, + total: n_chunks_total, + message: "a background thread panicked during post-loop tensor readback".to_string(), + }); + } + Ok(EvalOutput { predictions_daily, observations_daily, @@ -315,8 +366,34 @@ pub fn evaluate( #[cfg(test)] mod tests { - use super::corrupted_chunk_reason; + use super::{corrupted_chunk_reason, ensure_panic_hook_installed, take_worker_panicked}; use ndarray::Array2; + use std::sync::Mutex; + + // WORKER_PANICKED is a process-global static; serialize the two tests + // that touch it so they can't race against each other under the default + // parallel test harness. + static PANIC_FLAG_TEST_LOCK: Mutex<()> = Mutex::new(()); + + #[test] + fn worker_thread_panic_sets_flag() { + let _guard = PANIC_FLAG_TEST_LOCK.lock().unwrap(); + ensure_panic_hook_installed(); + take_worker_panicked(); // clear any residual state before asserting + + let result = std::thread::spawn(|| panic!("simulated cubecl worker panic")).join(); + assert!(result.is_err()); + assert!(take_worker_panicked(), "flag should be set after a background thread panic"); + assert!(!take_worker_panicked(), "flag should be cleared by the previous take"); + } + + #[test] + fn no_panic_leaves_flag_clear() { + let _guard = PANIC_FLAG_TEST_LOCK.lock().unwrap(); + ensure_panic_hook_installed(); + take_worker_panicked(); // clear any residual state + assert!(!take_worker_panicked()); + } #[test] fn healthy_chunk_is_not_corrupted() { From f322eb1b990e3fb5c3a174f6df50e942b860b65b Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Thu, 16 Jul 2026 14:35:22 -0400 Subject: [PATCH 4/8] fix(doctest): fence pseudocode formulas so rustdoc doesn't compile them Three doc comments embedded arrow-diagram/formula pseudocode as an indented block (4-space indent after `///`), which rustdoc's default doctest harness tries to compile as runnable Rust regardless of whether it's valid syntax. Pre-existing, unrelated to this session's other changes -- surfaced by running the full `cargo test --release` suite (deferred earlier in the session to avoid CPU contention with in-flight training runs). Wraps each in an explicit ```text fence so rustdoc treats it as documentation, not a test. `cargo test --release --doc` now passes (was 4 failing to compile). --- src/cuda_graph/geometry_kernel.rs | 12 +++++++++--- src/sparse/cusparse.rs | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cuda_graph/geometry_kernel.rs b/src/cuda_graph/geometry_kernel.rs index ea0fd22..e1562aa 100644 --- a/src/cuda_graph/geometry_kernel.rs +++ b/src/cuda_graph/geometry_kernel.rs @@ -10,7 +10,9 @@ //! (trapezoidal geometry + Muskingum coefficients) with 19 outputs. The //! captured forward region for SP-10 will be: //! -//! K1 → cuSPARSE SpMV → K2 (b_rhs) → assemble → cuSPARSE SpSV → K3 (clamp) +//! ```text +//! K1 → cuSPARSE SpMV → K2 (b_rhs) → assemble → cuSPARSE SpSV → K3 (clamp) +//! ``` //! //! K1 produces every saved-state intermediate that `TimestepOp::backward` //! consumes from the pre-SpMV part of the chain — i.e., everything in @@ -351,7 +353,9 @@ pub fn forward_k1_kernel( /// SP-10 Phase 2 K2: fused S25 of `forward_chain_inner`. /// -/// b_rhs = c2 * i_t + c3 * q_t + c4 * q_prime_t +/// ```text +/// b_rhs = c2 * i_t + c3 * q_t + c4 * q_prime_t +/// ``` /// /// Single elementwise linear combination over `[n_segments]`. `c2`, `c3`, `c4` /// come from K1; `i_t` is the cuSPARSE SpMV output; `qt`, `qpt` are per-step @@ -405,7 +409,9 @@ pub fn assemble_kernel( /// SP-10 Phase 2 K3: fused S28 of `forward_chain_inner`. /// -/// q_next = max(x_sol, discharge_lb) +/// ```text +/// q_next = max(x_sol, discharge_lb) +/// ``` /// /// Single elementwise lower-clamp over `[n_segments]`. `x_sol` is the /// cuSPARSE SpSV solve output. Mirrors `mmc_op.rs:653-654` diff --git a/src/sparse/cusparse.rs b/src/sparse/cusparse.rs index 81e01c7..c92a6cd 100644 --- a/src/sparse/cusparse.rs +++ b/src/sparse/cusparse.rs @@ -334,7 +334,9 @@ where /// GPU per-nnz grada scatter using pure BURN tensor ops: /// -/// grada[k] = -gradb[row_for_nnz[k]] * x[col[k]] +/// ```text +/// grada[k] = -gradb[row_for_nnz[k]] * x[col[k]] +/// ``` /// /// No custom CUDA kernel — `Tensor::select` (gather) compiles on any backend /// (NdArray for CPU tests, Cuda for the GPU path). This path runs entirely on From 532c4c2b18947ffab54c9f4bcb5eccebbf98f221 Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Thu, 16 Jul 2026 15:05:03 -0400 Subject: [PATCH 5/8] docs(skill): document the GPU eval OOM incident and fix in ddrs-run-and-operate Adds section 14 covering the cubecl-cuda silent-corruption failure mode (symptom, root cause, the fix from the two previous commits, and how to tell if your installed binary predates it), a new Common Failure Checklist row, a cross-reference from the Performance Baseline section noting alternative Q' stores score below the documented CONUS benchmark, and updated frontmatter/Provenance sections pointing at this session's findings docs and verification greps. --- .claude/skills/ddrs-run-and-operate/SKILL.md | 77 +++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/.claude/skills/ddrs-run-and-operate/SKILL.md b/.claude/skills/ddrs-run-and-operate/SKILL.md index 4e5a4dd..f4aaeb1 100644 --- a/.claude/skills/ddrs-run-and-operate/SKILL.md +++ b/.claude/skills/ddrs-run-and-operate/SKILL.md @@ -1,6 +1,6 @@ --- name: ddrs-run-and-operate -description: "Use when you need to install, configure, launch, or monitor a ddrs training/eval run; diagnose a stale binary; resume from a checkpoint; select a CUDA device; manage data-source groups; or understand workspace artifact layout. Also use when troubleshooting silent correctness failures caused by stale PATH binaries or CUDA graphs masking NaN." +description: "Use when you need to install, configure, launch, or monitor a ddrs training/eval run; diagnose a stale binary; resume from a checkpoint; select a CUDA device; manage data-source groups; or understand workspace artifact layout. Also use when troubleshooting silent correctness failures caused by stale PATH binaries, CUDA graphs masking NaN, or a GPU eval Phase 2 that spins at high CPU without exiting (cubecl worker-thread OOM panic — see §14)." --- # ddrs: Run and Operate @@ -305,6 +305,8 @@ Best trained result (precip-driven disagg + L1 loss, 2365 gauges, 2026-06-23): N Note: KGE does NOT beat the summed-Q' baseline in any config as of 2026-07-05. NSE beats it (+0.037 with precip disagg). This is a known open problem. +These numbers are for the default `merit_dhbv2_UH_retrospective.ic` streamflow store. Swapping in an alternative Q' store (dHBV AORC2F variants, NH LSTM daily/hourly) changes both the baseline and the trained result substantially — see `docs/2026-07-16-aorc2f-wave1-findings.md` / `docs/2026-07-16-wave2-cross-wave-findings.md` for a 4-store comparison (all four came in below this benchmark). + --- ## 11. Legacy Binaries (deprecated, removed in 0.4) @@ -346,6 +348,69 @@ Caveat (2026-06-06): the reference DDR state lives only on the local desktop's ` | Flat files `epoch_E_mb_M.mpk` in checkpoints/ | Stale pre-checkpoint-resume binary | `cargo install --path .` | | Hourly run uses flat repeat-24 | Missing `aorc_precip` source OR `use_precip: false` | Add `conus-hourly` source group AND set `use_precip: true` | | Dataset open error with `use_precip: true` | `aorc_precip` not in `data_sources` | `ddrs sources use conus-hourly` | +| `train-and-test`'s Phase 2 (testing) spins at 900%+ CPU, repeatedly logging `thread 'DSD-0-0' panicked ... can't allocate buffer` without ever exiting | cubecl-cuda background-worker OOM panic that doesn't propagate to the caller (Phase 2's per-chunk buffer, ~4.2 GB at `batch_size: 15` days, is far larger than a training minibatch's) | Kill the process; recover via the standalone `eval` binary against the last checkpoint with `--backend cpu` (see §14). Since 2026-07-16 a fresh binary hard-fails instead of spinning — reinstall if you see this. | + +--- + +## 14. GPU Eval OOM — Silent Corruption Bug (fixed 2026-07-16) + +**Symptom:** `--backend cuda` `train-and-test`/`eval` completes training fine, +then Phase 2 (testing) hits a CUDA OOM inside a cubecl-cuda background +worker thread (`thread 'DSD-0-0' panicked ... can't allocate buffer of size: +4178264064`). The panic does **not** propagate to the caller — cubecl's +server loop catches it, logs, and drops the task — so the main thread just +keeps looping through all 366 eval chunks, writing whatever half-written GPU +buffer is left. **This happens even with no other process on the GPU** — +Phase 1 alone can leave too little headroom for Phase 2's much larger +per-chunk buffer at the default `testing.batch_size: 15` (days/chunk) on a +16 GB card. Confirmed twice independently (2026-07-16, distributed-source +and daily-lstm-source arms), including once running solo. + +**Fixed as of 2026-07-16** (`src/training/eval.rs`): `evaluate()` now +detects this via two layered checks and returns +`DataError::CorruptedEvalChunk` instead of silently continuing — +1. `corrupted_chunk_reason` — all-zero or non-finite chunk values (kept as + defense-in-depth; alone it MISSED the second incident because the + corrupted chunk happened to contain plausible-looking stale finite + values). +2. `WORKER_PANICKED` — a process-global flag set by a custom panic hook + (`ensure_panic_hook_installed`), checked after every chunk and once more + after the post-loop tensor readback. This is the reliable detector — it + fires on ANY background-thread panic regardless of what ends up in the + output buffer. + +A binary built before 2026-07-16 will still exhibit the silent-spin +behavior — `cargo install --path .` to pick up the fix (see the +stale-binary trap in §1). 6 unit tests cover both detectors: +`cargo test --lib training::eval::tests`. + +**Practical recommendation:** on hardware where this triggers, either (a) +default to `--backend cpu` for `train-and-test`/`eval` workflows (training +alone is fine on GPU), or (b) try lowering `testing.batch_size` (days/chunk) +to shrink the per-chunk buffer — not yet validated as a GPU fix. **Recovery +without retraining:** the standalone `eval` binary reads any completed +checkpoint directly and supports `--backend cpu` independent of what backend +Phase 1 used: + +```bash +cargo install --path . # only if your eval binary predates the fix +~/.cargo/bin/eval --config /config.yaml \ + --checkpoint /checkpoints/epoch_E_mb_M/head \ + --output /eval/predictions.zarr \ + --backend cpu +``` + +**Running two arms in parallel on one GPU:** don't co-schedule two +`--backend cuda` trainings — Phase 1 alone uses ~9.9 GB on a 16 GB card, so +a second concurrent process reliably OOMs within seconds. Give each arm a +different backend from the start (one `--backend cuda`, one `--backend +cpu`) rather than launching both on GPU and recovering after a crash. See +`config/experiments/aorc2f_distributed_frozen_chunk1.yaml` / +`aorc2f_lumped_frozen_chunk1.yaml` / `lstm_daily_frozen_chunk1.yaml` / +`lstm_hourly_native.yaml` and +`docs/2026-07-16-aorc2f-wave1-findings.md` / +`docs/2026-07-16-wave2-cross-wave-findings.md` for a full worked campaign +(4 CONUS train-and-test arms, 2 OOM incidents, full recovery). --- @@ -372,6 +437,16 @@ ls /home/tbindas/projects/ddrs/config/sources/ # Re-verify stale-binary trap documentation: grep -A 20 "STALE-BINARY TRAP" /home/tbindas/projects/ddrs/CLAUDE.md + +# Re-verify the GPU eval OOM fix (§14): +grep -n "CorruptedEvalChunk\|WORKER_PANICKED" /home/tbindas/projects/ddrs/src/training/eval.rs /home/tbindas/projects/ddrs/src/data/error.rs ``` Volatile facts dated 2026-07-05: baseline metrics, best-run metrics, leakance GO verdict, K_D ceiling observation, KGE vs baseline status. Re-check these after any new experiment campaign. + +Volatile facts dated 2026-07-16: the GPU eval OOM bug (§14) and its fix; the +4-arm AORC2F/LSTM campaign benchmark numbers in +`docs/2026-07-16-aorc2f-wave1-findings.md` and +`docs/2026-07-16-wave2-cross-wave-findings.md`. Re-check the OOM fix is +still present (grep above) before trusting a GPU `train-and-test` run's +Phase 2 output on new hardware. From 5c28893eeb2bd3695c3e1c619fdaa8099873161f Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Thu, 16 Jul 2026 20:28:11 -0400 Subject: [PATCH 6/8] docs: AORC2F/LSTM Q' campaign findings + cross-wave comparison + disagg sensitivity Four CONUS train-and-test arms against alternative Q' forecast stores (AORC2F distributed, AORC2F lumped, daily-lstm, hourly-lstm), all using the frozen capacity-boosted disagg head where applicable, compared against each arm's own summed-Q' baseline and the standing 0.7152/0.7106 CONUS benchmark: arm trained NSE/KGE own baseline NSE/KGE AORC2F distributed 0.3437 / 0.3256 0.233 / 0.327 AORC2F lumped 0.5259 / 0.5175 -0.105 / 0.393 daily-lstm 0.5674 / 0.6169 0.437 / 0.616 hourly-lstm 0.5543 / 0.4852 0.532 / 0.547 None of the four stores beat the standing benchmark under this routing configuration; daily-lstm comes closest. Also includes an input-perturbation check confirming the frozen disagg head is byte-identical and genuinely precip-responsive across all three arms that load it (expected, since all three warm-start from the same pretrained checkpoint with freeze: true). scripts/wave_comparison_plots.py generates the NSE/KGE box plot and CDFs across all 4 trained arms + their baselines (output/2026-07-16-wave-comparison/). --- docs/2026-07-16-aorc2f-wave1-findings.md | 94 +++++++++++++ ...-07-16-disagg-head-sensitivity-findings.md | 87 +++++++++++++ docs/2026-07-16-wave2-cross-wave-findings.md | 123 ++++++++++++++++++ scripts/wave_comparison_plots.py | 120 +++++++++++++++++ 4 files changed, 424 insertions(+) create mode 100644 docs/2026-07-16-aorc2f-wave1-findings.md create mode 100644 docs/2026-07-16-disagg-head-sensitivity-findings.md create mode 100644 docs/2026-07-16-wave2-cross-wave-findings.md create mode 100644 scripts/wave_comparison_plots.py diff --git a/docs/2026-07-16-aorc2f-wave1-findings.md b/docs/2026-07-16-aorc2f-wave1-findings.md new file mode 100644 index 0000000..f1b3389 --- /dev/null +++ b/docs/2026-07-16-aorc2f-wave1-findings.md @@ -0,0 +1,94 @@ +# AORC2F Q′ wave 1 — frozen capacity-boosted disagg head, findings (2026-07-16) + +Plan: `.claude/plans/glowing-bubbling-sprout.md` (agent-authored, not checked in) +Configs: `config/experiments/aorc2f_distributed_frozen_chunk1.yaml`, + `config/experiments/aorc2f_lumped_frozen_chunk1.yaml` +Prior benchmark: `docs/2026-06-23-precip-disaggregation-findings.md` (median NSE 0.7152 / KGE 0.7106) + +## 1. What this tests + +Two new dHBV rainfall-runoff Q′ forecast stores (distinct from the standard +`merit_dhbv2_UH_retrospective.ic` used in all prior CONUS benchmarks), routed +through the same CONUS KAN head + the 2026-07-10/12 capacity-boosted frozen +disagg head (`hidden 16, 2 layers, grid 20, k 3, chunk_days 1`, warm-started +from `output/disagg_pretrain/capacity_chunk1.mpk`, frozen). Everything else — +network, attributes, gauges, seed 42, 5 epochs, rho 90, warmup 5, L1 loss, no +leakance, train 1981/10/01–1995/09/30, eval 1995/10/01–2010/09/30 — matches +`kan_disagg_conus_frozen_chunk1.yaml`. + +| Arm | Model | Q′ store | Upstream checkpoint | Divides | +|---|---|---|---|---| +| Distributed | Distributed + UH routing (dHBV AORC2F v2) | `daily_dhbv_aorc2f_merit_unit_catchments.ic` | `CONUS2717_AORC2F_v2` ep69 | 197,088 | +| Lumped | Lumped (AORC) dHBV | `daily_dhbv2_merit_unit_catchments.ic` | `CONUS2717_AORC2F_LUMPED` ep63 | 197,088 | + +The "checkpoint" column is provenance only — these are DDR/PyTorch `.pt` +checkpoints of the upstream rainfall-runoff model that produced each store's +Q′ values; `ddrs` reads the resulting streamflow directly and never loads +those files. + +## 2. Execution notes (both arms hit a real GPU OOM — see below) + +Both arms were launched in parallel with `--backend cuda` (KAN head + router +both on GPU, per instruction). The card is a single 16 GB RTX 4080 Super. + +- **Lumped** hit a CUDA OOM ~30s into epoch 1 (`can't allocate buffer of size: + 16321536`, the *distributed* arm having claimed ~9.9 GB first). Relaunched + whole with `--backend cpu` (forces `sparse_solver=cpu` for both KAN and + router, not a YAML edit) — completed cleanly end-to-end on CPU. Total wall + time 02:23:20Z→10:49:20Z ≈ **8h 26min** (train ~1h 46min + test ~6h 40min). +- **Distributed** trained cleanly on GPU for all 5 epochs, but Phase 2 + (testing) hit a *silent* failure mode: a cubecl-cuda background worker + thread panicked on OOM (`can't allocate buffer of size: 4178264064` — Phase + 2's per-chunk buffer is ~260x a training minibatch's) without the panic + propagating to the main thread, which kept looping through eval chunks + indefinitely at 900%+ CPU instead of failing. Killed and recovered by + running the (deprecated) standalone `eval` binary directly against the + epoch-5 checkpoint with `--backend cpu` — this is why the distributed arm + has no `manifest.json` (the killed `ddrs run` process never wrote one) but + does have `checkpoints/`, `config.yaml`, and `eval/predictions.zarr`. +- This silent-corruption failure mode is now hard-gated in `src/training/eval.rs` + (`DataError::CorruptedEvalChunk` — see commit introducing + `corrupted_chunk_reason`); a future run in the same situation will fail + loudly at the first corrupted chunk instead of producing bad output. +- **Lesson for future waves:** don't co-launch two `--backend cuda` CONUS + trainings on one 16 GB card — Phase 1 alone uses ~9.9 GB, leaving no + headroom for a second process, and Phase 2's larger buffers make it worse. + Wave 2 launches one GPU + one CPU arm from the start instead of + crash-and-retry. + +## 3. Results + +| Arm | Trained median NSE | Trained median KGE | Own summed-Q′ baseline NSE | Own baseline KGE | Δ NSE vs own baseline | Δ KGE vs own baseline | +|---|---|---|---|---|---|---| +| Distributed | 0.3437 | 0.3256 | 0.233 | 0.327 | +0.111 | −0.001 | +| Lumped | 0.5259 | 0.5175 | −0.105 | 0.393 | +0.631 | +0.124 | + +(2365/2365 gauges finite NSE in both arms.) + +Both are well below the standing CONUS benchmark (median NSE 0.7152 / KGE +0.7106, `merit_dhbv2_UH_retrospective.ic` streamflow, same disagg head) — +**Δ vs benchmark: distributed −0.372 NSE / −0.385 KGE; lumped −0.189 NSE / +−0.193 KGE.** + +Lumped clearly outperforms distributed on this routing setup (+0.182 NSE, ++0.192 KGE) and its raw Q′ is dramatically improved by routing (own baseline +NSE −0.105 → trained 0.526), whereas distributed's raw Q′ was already +directionally reasonable (baseline NSE 0.233) and routing helps far less in +relative terms. + +**Backend caveat:** distributed trained on GPU, tested on CPU (via the +recovery path); lumped ran entirely on CPU. Per prior parity testing +(`tests/sparse_cusparse_v5.rs`), CPU/CUDA numerical drift is ULP-scale +(~1e-4 rel tolerance) — far smaller than the 0.18 NSE gap observed here, so +the lumped > distributed ranking is not a backend artifact. + +## 4. Interpretation + +Neither AORC2F Q′ store is a drop-in replacement for the standard +`merit_dhbv2_UH_retrospective.ic` under this routing configuration — both +lose substantial skill relative to the standing benchmark. Of the two, the +lumped (AORC) dHBV variant is the better Q′ source for this KAN+disagg +routing setup. Follow-up (not yet done): check whether the AORC2F stores' +lower coverage assumptions, different training windows, or different +input-forcing pipeline (vs the retrospective store) explain the gap before +concluding this is a routing-model limitation. diff --git a/docs/2026-07-16-disagg-head-sensitivity-findings.md b/docs/2026-07-16-disagg-head-sensitivity-findings.md new file mode 100644 index 0000000..15e2e4a --- /dev/null +++ b/docs/2026-07-16-disagg-head-sensitivity-findings.md @@ -0,0 +1,87 @@ +# Input perturbation experiment — frozen disagg head sensitivity (2026-07-16) + +Example: `examples/kan_disagg_trained_sensitivity.rs` +Checkpoints probed: `.ddrs/runs/2026-07-16T02-22-14Z-train-and-test`, +`.ddrs/runs/2026-07-16T02-23-20Z-train-and-test`, +`.ddrs/runs/2026-07-16T11-31-50Z-train-and-test`'s +`checkpoints/epoch_5_mb_35/head` (all three train-and-test arms that used +the frozen capacity-boosted disagg head — `docs/2026-07-16-aorc2f-wave1-findings.md`, +`docs/2026-07-16-wave2-cross-wave-findings.md`; run-ID-to-experiment mapping +in `.ddrs/README.md`). + +## 1. Setup + +Fixed daily Q' = 10 m³/s. Two sweeps against the loaded `DisaggHead`: +1. **Intensity sweep**: inject a precip spike (0.1, 1, 2, 4, 8, 16 mm/hr) at + a fixed hour (12), background 0.1 mm/hr elsewhere; record the resulting + 24-hour disaggregated shape (`*_trained.csv`). +2. **Hour-position sweep**: fix intensity (8 mm/hr), vary WHICH hour (0, 3, + 6, ..., 21) receives the spike; record the shape (`*_hour_position.csv`). + +Outputs: `/plots/precip_sensitivity_trained.csv` and +`..._hour_position.csv` for each of the 3 runs. + +## 2. All three runs' outputs are byte-identical — expected, and confirmed + +`diff` across all 3 runs' CSVs reports zero differences. This is the +*correct* result, not a bug: all three configs warm-start the disagg head +from the same `output/disagg_pretrain/capacity_chunk1.mpk` with `freeze: +true`, so the head's weights are literally identical across arms regardless +of which Q' model or routing outcome each arm produced. This confirms +`freeze: true` behaved as documented — no accidental per-run drift — and +that wave 1/2's cross-arm NSE/KGE differences are attributable entirely to +the routing KAN head + Q' source, not to any variation in the disagg head. + +## 3. Sensitivity behavior (from the distributed arm's CSVs — identical for all 3) + +**Mass balance holds exactly** at every intensity and every injected hour: +`sum(hourly_value over 24h) == 240.0` (= 10 m³/s × 24h), confirming the +mass-preserving constraint survives the frozen/warm-start path. + +**Intensity sweep** — output peak hour and magnitude both respond to the +injected spike (not a static shape): + +| precip intensity (mm/hr) | peak hour | peak value (m³/s) | +|---|---|---| +| 0.1 | 1 | 11.49 | +| 1.0 | 1 | 11.18 | +| 2.0 | 2 | 11.05 | +| 4.0 | 6 | 10.51 | +| 8.0 | 19 | 11.29 | +| 16.0 | 17 | 15.19 | + +**Hour-position sweep** — output peak hour tracks the injected storm hour +loosely but not with a simple fixed lag; the relationship is non-monotonic +(e.g. storm at hour 15 → output peak at hour 6, storm at hour 21 → peak at +hour 23): + +| swept storm hour | output peak hour | peak value (m³/s) | +|---|---|---| +| 0 | 4 | 11.89 | +| 3 | 6 | 11.08 | +| 6 | 10 | 11.88 | +| 9 | 19 | 11.46 | +| 12 | 19 | 11.29 | +| 15 | 6 | 10.83 | +| 18 | 5 | 11.01 | +| 21 | 23 | 12.43 | + +## 4. Interpretation + +The frozen head is genuinely precip-responsive (peak timing/magnitude vary +with both intensity and injection hour) rather than reproducing a fixed +diurnal shape, and it's exactly mass-conserving by construction. The +peak-hour response to storm timing is not a simple causal lag — expected, +since the KAN maps `(daily_q, precip[24]) → hourly[24]` as a joint function +over the whole day rather than a strictly-causal per-hour convolution; this +matches how it was pretrained (`pretrain_disagg_capacity`) and is consistent +with the companion mass-balance example +(`examples/kan_disagg_mass_balance_real.rs`). No further action needed — +this was a confirmatory check, not a discovery of a defect. + +**Caveat on the hour-position sweep:** at the fixed 8 mm/hr spike used there, +the response shape is nearly flat (peaks 10.8–11.9 m³/s vs a 10.0 m³/s +uniform baseline) — the erratic peak-hour jumps (e.g. 15→6, 18→5) are mostly +an argmax over a low-amplitude, noise-sensitive shape rather than meaningful +timing physics. The genuinely strong, clearly-structured timing response +only shows up at the higher 16 mm/hr intensity in the intensity sweep. diff --git a/docs/2026-07-16-wave2-cross-wave-findings.md b/docs/2026-07-16-wave2-cross-wave-findings.md new file mode 100644 index 0000000..9ff8300 --- /dev/null +++ b/docs/2026-07-16-wave2-cross-wave-findings.md @@ -0,0 +1,123 @@ +# LSTM Q′ wave 2 + cross-wave comparison, findings (2026-07-16) + +Plan: `.claude/plans/glowing-bubbling-sprout.md` (agent-authored, not checked in) +Wave 1: `docs/2026-07-16-aorc2f-wave1-findings.md` +Configs: `config/experiments/lstm_daily_frozen_chunk1.yaml`, + `config/experiments/lstm_hourly_native.yaml` +Plots: `output/2026-07-16-wave-comparison/` (script: `scripts/wave_comparison_plots.py`) + +## 1. What this tests + +Two NH-LSTM Q′ forecast stores routed through the CONUS KAN head, same +constants as wave 1 (seed 42, 5 epochs, rho 90, warmup 5, L1 loss, no +leakance, train 1981/10/01–1995/09/30, eval 1995/10/01–2010/09/30): + +| Arm | Q′ store | Resolution | Disagg head | +|---|---|---|---| +| daily-lstm | `daily_lstm_merit_unit_catchments.ic` (288,421 divides) | daily | frozen capacity-boosted (same as wave 1) | +| hourly-lstm | `hourly_lstm_merit_unit_catchments.ic` (197,088 divides) | hourly-native | none — hourly-native + disagg head is a hard config error | + +daily-lstm updates the older `equif_daily_lstm_disagg.yaml` arm (2026-07-07 +equifinality campaign), which pre-dates the 2026-07-10/12 frozen +capacity-boosted disagg-head fixes and used a plain, jointly-trained +`hidden_size: 16` head. + +## 2. Execution — this time, GPU alone (no contention) also OOM'd + +Lesson from wave 1 was "don't co-schedule two `--backend cuda` trainings" — +so wave 2 launched **one arm per backend from the start**: daily-lstm on +GPU alone, hourly-lstm on CPU alone. hourly-lstm ran clean end-to-end on CPU +(median NSE 0.5543, KGE 0.4852, 2365/2365 finite). + +**daily-lstm hit the exact same silent-corruption incident as wave 1's +distributed arm — with NO other process on the GPU.** Training completed +cleanly (5 epochs, all checkpoints written). Phase 2 (testing) then hit the +same cubecl-cuda `DSD-0-0` worker-thread OOM panic (`can't allocate buffer of +size: 4178264064`) that never propagates to the caller; the process kept +looping through chunks (confirmed to chunk 6/366) at 900%+ CPU. **This +confirms the failure is NOT contention-specific** — Phase 1 alone leaves the +16 GB card with too little headroom for Phase 2's ~4.2 GB per-chunk buffer, +even running solo. Recovery: killed, then ran the standalone `eval` binary +against the epoch-5 checkpoint with `--backend cpu` (same recovery pattern as +wave 1). + +**Practical recommendation going forward:** on this GPU, either (a) always +use `--backend cpu` for `train-and-test`/eval workflows at the default +`testing.batch_size: 15` (days/chunk), or (b) reduce `testing.batch_size` to +shrink the per-chunk buffer (e.g. 3-5 days) and re-test whether Phase 2 then +fits on GPU — not yet tried. GPU is fine for training only (`--workflow +train`) at these settings. + +### Bug found and fixed mid-campaign + +The corruption-detection gate added after wave 1's incident +(`corrupted_chunk_reason` — all-zero/non-finite value check) did **not** +catch this second occurrence: the corrupted chunk's buffer held stale GPU +memory that happened to look like plausible finite, non-zero predictions. +Strengthened with a process-global panic-hook detector +(`WORKER_PANICKED`/`ensure_panic_hook_installed`/`take_worker_panicked` in +`src/training/eval.rs`) that catches ANY background-thread panic during a +chunk or the post-loop tensor readback, independent of what the output +values look like. See `docs/2026-07-16-aorc2f-wave1-findings.md` and the +commit introducing `DataError::CorruptedEvalChunk` for the first (partial) +fix; 6 unit tests cover both detectors (`cargo test --lib training::eval`). + +## 3. Results + +| Arm | Trained median NSE | Trained median KGE | Own baseline NSE | Own baseline KGE | +|---|---|---|---|---| +| daily-lstm | 0.5674 | 0.6169 | 0.437 | 0.616 | +| hourly-lstm | 0.5543 | 0.4852 | 0.532 | 0.547 | + +(2365/2365 gauges finite NSE in both arms.) + +daily-lstm is the best-performing arm in this entire campaign on NSE (0.5674) +and its KGE (0.6169) roughly matches its own baseline (+0.001) — routing +doesn't hurt here. hourly-lstm's routed KGE (0.4852) is noticeably *worse* +than its own raw-Q′ baseline (0.547, Δ −0.062) despite an NSE gain (+0.022) — +the routing model is trading KGE's variance/bias components for +correlation/NSE here, worth a follow-up if the hourly-native arm is pursued +further. + +## 4. Cross-wave comparison (all 4 arms) + +| Arm | Trained NSE | Trained KGE | Own baseline NSE | Own baseline KGE | Δ vs 0.7152/0.7106 benchmark (NSE/KGE) | +|---|---|---|---|---|---| +| AORC2F distributed | 0.3437 | 0.3256 | 0.233 | 0.327 | −0.372 / −0.385 | +| AORC2F lumped | 0.5259 | 0.5175 | −0.105 | 0.393 | −0.189 / −0.193 | +| daily-lstm | 0.5674 | 0.6169 | 0.437 | 0.616 | −0.148 / −0.094 | +| hourly-lstm | 0.5543 | 0.4852 | 0.532 | 0.547 | −0.161 / −0.225 | + +**Ranking by trained NSE:** daily-lstm (0.567) > lumped (0.526) > hourly-lstm +(0.554, close 3rd) > distributed (0.344). By trained KGE: daily-lstm (0.617) +> lumped (0.518) > hourly-lstm (0.485) > distributed (0.326). + +**None of the four new Q′ stores match the standing CONUS benchmark** +(median NSE 0.7152 / KGE 0.7106, `merit_dhbv2_UH_retrospective.ic`) under +this exact routing configuration — daily-lstm comes closest (−0.148 NSE, +−0.094 KGE) but all four arms lose meaningful skill relative to it. The +routing model consistently helps most on arms with weak/negative raw-Q′ +baselines (lumped: baseline NSE −0.105 → trained 0.526; distributed: +baseline 0.233 → trained 0.344) and helps least — or even hurts on KGE — on +arms whose raw Q′ is already strong (hourly-lstm, daily-lstm). + +Full distribution comparison (8 series: 4 trained + 4 own baselines): +`output/2026-07-16-wave-comparison/wave_comparison_boxplot.png`, +`wave_comparison_nse_cdf.png`, `wave_comparison_kge_cdf.png`. Note: +`ddr.validation.plot_cdf`'s legend hardcodes the label `NSE=` +regardless of which metric was passed in — the KGE CDF's legend text says +"NSE=" but is plotting KGE medians (matches the title/axis, which are +correct). Cosmetic upstream-DDR quirk, not a data error. + +## 5. Open follow-ups + +- Try `testing.batch_size` < 15 days to see if Phase 2 becomes GPU-viable + (would meaningfully speed up future campaigns — each of these runs took + 2–8.5 hours, dominated by the 366-chunk CPU testing phase). +- hourly-lstm's KGE regression under routing (routed 0.485 vs baseline 0.547) + is unexplained — worth a per-gauge diff against distributed/lumped/daily + to see if it's concentrated in a subset (e.g. high-variance flashy basins). +- None of these four Q′ stores currently beat the standard + `merit_dhbv2_UH_retrospective.ic` benchmark; before concluding routing + can't help them further, check whether their coverage/fill-rate or + training-window alignment differs materially from the benchmark store. diff --git a/scripts/wave_comparison_plots.py b/scripts/wave_comparison_plots.py new file mode 100644 index 0000000..abebccb --- /dev/null +++ b/scripts/wave_comparison_plots.py @@ -0,0 +1,120 @@ +"""Cross-run NSE/KGE comparison plots for the 2026-07-16 AORC2F + LSTM wave 1/2 +campaign (docs/2026-07-16-aorc2f-wave1-findings.md, +docs/2026-07-16-wave2-cross-wave-findings.md). + +Run from ddrs-py's venv: + cd ddrs-py && uv run python ../scripts/wave_comparison_plots.py +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import numpy as np + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / ".claude/skills/ddrs-eval-plots/scripts")) +from load_ddrs_predictions import load_predictions_zarr # noqa: E402 + +from ddr.validation import Metrics, plot_box_fig, plot_cdf # noqa: E402 + +OUT_DIR = REPO / "output" / "2026-07-16-wave-comparison" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +# Run-ID-to-experiment mapping: .ddrs/README.md (all 4 runs and their +# baseline cache keys were consolidated from separate parallel-launch +# workspaces into the main .ddrs/ workspace after the campaign finished). +RUNS = [ + { + "label": "AORC2F distributed", + "pred_zarr": REPO / ".ddrs/runs/2026-07-16T02-22-14Z-train-and-test/eval/predictions.zarr", + "baseline_dir": REPO / ".ddrs/baselines/86b907e4b17de998", + }, + { + "label": "AORC2F lumped", + "pred_zarr": REPO / ".ddrs/runs/2026-07-16T02-23-20Z-train-and-test/eval/predictions.zarr", + "baseline_dir": REPO / ".ddrs/runs/2026-07-16T02-23-20Z-train-and-test/baseline", + }, + { + "label": "daily-lstm", + "pred_zarr": REPO / ".ddrs/runs/2026-07-16T11-31-50Z-train-and-test/eval/predictions.zarr", + "baseline_dir": REPO / ".ddrs/baselines/7ffd117fc9161734", + }, + { + "label": "hourly-lstm", + "pred_zarr": REPO / ".ddrs/runs/2026-07-16T11-31-52Z-train-and-test/eval/predictions.zarr", + "baseline_dir": REPO / ".ddrs/runs/2026-07-16T11-31-52Z-train-and-test/baseline", + }, +] + + +def load_baseline_f32(bdir: Path): + man = json.load(open(bdir / "manifest.json")) + g, t = man["n_gauges"], man["n_days"] + pred = np.fromfile(bdir / "predictions.f32", dtype="12}{'median KGE':>12}") +for lbl, r in zip(labels, results): + d = dict(r) + nse = np.nanmedian(d["nse"]) + kge = np.nanmedian(d["kge"]) + print(f"{lbl:<28}{nse:>12.4f}{kge:>12.4f}") From 4c147f841f173a69600ff13164b5b0f1871fbb7a Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Mon, 20 Jul 2026 21:51:10 -0400 Subject: [PATCH 7/8] docs: correct backend-mismatch dismissal; flag double-routing hypothesis + missing controls A 2026-07-21 adversarial review (fable subagent, prompted by the user flagging the AORC2F-distributed arm's result as suspicious) found the wave 1 doc's dismissal of the GPU-train/CPU-eval backend mismatch cited an inapplicable test (tests/sparse_cusparse_v5.rs bounds one forward+backward solve, not 5 epochs of Adam-driven divergence). Replaces it with the actual decisive evidence: mb=0 training loss, logged before any weight update from the identical seed-42 init, already orders all four arms exactly as the final results do -- the ranking is data-borne, not a training-backend artifact. Adds the review's leading untested hypothesis: the "distributed + UH routing" AORC2F store may already have unit-hydrograph routing applied upstream, so ddrs's Muskingum-Cunge routing on top would double-route it -- consistent with both its modest-but-not-negative raw baseline and its below-the-rest routing gain. Also records the review's list of missing controls (no replicate seeds, no backend-consistent control run, no per-gauge/spatial diagnostics, benchmark reference predates mid-campaign eval-code changes). --- docs/2026-07-16-aorc2f-wave1-findings.md | 55 ++++++++++++++++---- docs/2026-07-16-wave2-cross-wave-findings.md | 9 ++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/docs/2026-07-16-aorc2f-wave1-findings.md b/docs/2026-07-16-aorc2f-wave1-findings.md index f1b3389..e1ff4b0 100644 --- a/docs/2026-07-16-aorc2f-wave1-findings.md +++ b/docs/2026-07-16-aorc2f-wave1-findings.md @@ -76,11 +76,22 @@ NSE −0.105 → trained 0.526), whereas distributed's raw Q′ was already directionally reasonable (baseline NSE 0.233) and routing helps far less in relative terms. -**Backend caveat:** distributed trained on GPU, tested on CPU (via the -recovery path); lumped ran entirely on CPU. Per prior parity testing -(`tests/sparse_cusparse_v5.rs`), CPU/CUDA numerical drift is ULP-scale -(~1e-4 rel tolerance) — far smaller than the 0.18 NSE gap observed here, so -the lumped > distributed ranking is not a backend artifact. +**Backend caveat — corrected 2026-07-21.** An earlier version of this note +dismissed the distributed-trained-on-GPU / lumped-trained-on-CPU mismatch by +citing `tests/sparse_cusparse_v5.rs`'s ULP-scale (~1e-4 rel) single-timestep +bit-match tolerance. That citation is not valid evidence here: it bounds one +forward+backward solve, not 5 epochs of Adam-driven weight divergence plus a +15-year autoregressive eval rollout, where compounding numerical drift could +plausibly matter. The correct, decisive evidence was sitting in the run +logs and wasn't checked at the time: **mb=0 training loss — logged before +any weight update, with both arms starting from the identical seed-42 KAN +initialization — already orders the arms exactly as the final results do** +(distributed 12.29 > lumped 9.22 > daily-lstm 7.32 ≈ hourly-lstm 7.29, see +`.ddrs/runs/*/run.log`). Since this ordering exists before any +backend-dependent training has occurred, the distributed-vs-lumped gap is +data-borne, not a training-backend artifact — though a same-seed +`--backend cpu` re-run of the distributed arm (not yet done) would be the +direct confirmation. ## 4. Interpretation @@ -88,7 +99,33 @@ Neither AORC2F Q′ store is a drop-in replacement for the standard `merit_dhbv2_UH_retrospective.ic` under this routing configuration — both lose substantial skill relative to the standing benchmark. Of the two, the lumped (AORC) dHBV variant is the better Q′ source for this KAN+disagg -routing setup. Follow-up (not yet done): check whether the AORC2F stores' -lower coverage assumptions, different training windows, or different -input-forcing pipeline (vs the retrospective store) explain the gap before -concluding this is a routing-model limitation. +routing setup. + +**Leading untested hypothesis (added 2026-07-21):** the distributed store +is labeled "Distributed + UH routing" — i.e. its upstream dHBV model may +already apply unit-hydrograph routing (temporal lag/attenuation) to each +divide's Q′ before ddrs ever sees it. `docs/nh-qprime-store-contract.md` +requires Qr to be local lateral inflow with **no routing already applied**; +`ddrs import --dry-run` validates schema/coverage/units, not this semantic +contract, so a pre-routed store would pass import validation silently. If +true, this would explain both halves of the anomaly: (a) the modest-but-not- +negative raw baseline (0.233 NSE, vs lumped's fully-unrouted −0.105) is +consistent with Q′ that already carries some routing signal, and (b) MC +routing on top would double-route it — attenuating/lagging an +already-attenuated/lagged signal — explaining why distributed gains the +*least* from routing (+0.111 NSE) of all four arms despite starting from +the weakest raw NSE. Not yet tested: cross-correlate distributed vs lumped +Q′ at shared divides, and summed distributed Q′ vs USGS peak timing, to +check for an extra lag signature. + +**Other gaps identified in a 2026-07-21 adversarial review of this +campaign** (`/tmp/experiment-handoff-aorc2f-lstm-routing.md`): no +replicate seeds per arm (single seed=42 run each, so run-to-run noise is +unquantified — the 0.041 daily-lstm-vs-hourly-lstm gap is plausibly within +noise, the 0.18+ distributed gap probably is not, but neither is +rigorously bounded); no backend-consistent control run for the distributed +arm (never run fully CPU or fully GPU end-to-end); the 0.7152/0.7106 +reference benchmark predates two mid-campaign changes to the eval code +path, so the "vs benchmark" deltas compare across code states, not a +frozen baseline; no per-gauge/spatial diagnostics or KGE α/β/r +decomposition — all four arms were compared only by scalar medians. diff --git a/docs/2026-07-16-wave2-cross-wave-findings.md b/docs/2026-07-16-wave2-cross-wave-findings.md index 9ff8300..265239e 100644 --- a/docs/2026-07-16-wave2-cross-wave-findings.md +++ b/docs/2026-07-16-wave2-cross-wave-findings.md @@ -111,6 +111,15 @@ correct). Cosmetic upstream-DDR quirk, not a data error. ## 5. Open follow-ups +- **2026-07-21 correction:** wave 1's AORC2F-distributed-vs-lumped + backend-mismatch dismissal was based on an inapplicable test tolerance; + see the corrected §3 discussion in `docs/2026-07-16-aorc2f-wave1-findings.md` + for the actual (mb=0 loss ordering) evidence and the leading untested + hypothesis (the distributed store may already be UH-routed, causing + double-routing). An adversarial review of this whole campaign's controls + is at `/tmp/experiment-handoff-aorc2f-lstm-routing.md` — no replicate + seeds, no backend-consistent control run, and no per-gauge/spatial + diagnostics were done for any of the 4 arms. - Try `testing.batch_size` < 15 days to see if Phase 2 becomes GPU-viable (would meaningfully speed up future campaigns — each of these runs took 2–8.5 hours, dominated by the 366-chunk CPU testing phase). From f26409a313e24d5f59cb9250da3db5d768941238 Mon Sep 17 00:00:00 2001 From: Tadd Bindas Date: Mon, 20 Jul 2026 21:56:44 -0400 Subject: [PATCH 8/8] docs: test double-routing hypothesis against the store; mixed result Read Song, Bindas, Shen et al. (2025, WRR) -- the paper this model family descends from -- which clarifies that their own MC-routing arm is fed q_m (pre-aggregation, pre-UH-routing per-unit runoff), not q_b (the UH-routed, one-value-per-gage-basin streamflow). Directly tested the naive version of the double-routing hypothesis against daily_dhbv_aorc2f_merit_unit_catchments.ic: pulled Qr for 20 COMIDs spanning a 578-reach gauge subgraph (USGS 01184000) via icechunk/zarr and checked for bit-identical pairs, since q_b is architecturally incapable of native per-unit resolution. Result: 0/190 identical pairs in either the distributed or lumped store -- refutes the strong form (replicated basin-outlet flow). A secondary smoothness check (lag-1 autocorrelation, day-to-day volatility) found distributed somewhat smoother than lumped, a weak and inconclusive signal for a milder partial- routing variant. Recommends checking the actual export script's semantics directly rather than continuing to infer from store contents alone. --- docs/2026-07-16-aorc2f-wave1-findings.md | 56 +++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/docs/2026-07-16-aorc2f-wave1-findings.md b/docs/2026-07-16-aorc2f-wave1-findings.md index e1ff4b0..f5631c0 100644 --- a/docs/2026-07-16-aorc2f-wave1-findings.md +++ b/docs/2026-07-16-aorc2f-wave1-findings.md @@ -101,22 +101,46 @@ lose substantial skill relative to the standing benchmark. Of the two, the lumped (AORC) dHBV variant is the better Q′ source for this KAN+disagg routing setup. -**Leading untested hypothesis (added 2026-07-21):** the distributed store -is labeled "Distributed + UH routing" — i.e. its upstream dHBV model may -already apply unit-hydrograph routing (temporal lag/attenuation) to each -divide's Q′ before ddrs ever sees it. `docs/nh-qprime-store-contract.md` -requires Qr to be local lateral inflow with **no routing already applied**; -`ddrs import --dry-run` validates schema/coverage/units, not this semantic -contract, so a pre-routed store would pass import validation silently. If -true, this would explain both halves of the anomaly: (a) the modest-but-not- -negative raw baseline (0.233 NSE, vs lumped's fully-unrouted −0.105) is -consistent with Q′ that already carries some routing signal, and (b) MC -routing on top would double-route it — attenuating/lagging an -already-attenuated/lagged signal — explaining why distributed gains the -*least* from routing (+0.111 NSE) of all four arms despite starting from -the weakest raw NSE. Not yet tested: cross-correlate distributed vs lumped -Q′ at shared divides, and summed distributed Q′ vs USGS peak timing, to -check for an extra lag signature. +**Double-routing hypothesis — partially tested 2026-07-21, mixed result.** +The distributed store is labeled "Distributed + UH routing." Reading Song, +Bindas, Shen et al. (2025, WRR, 10.1029/2024WR038928 — the δHBV2.0/δMC +paper this campaign's model family descends from) clarifies the exact +mechanism: δHBV2.0UH runs HBV per MERIT unit basin to get `q_m` (distinct +per unit), **sums** all upstream units' `q_m` into one value per gage basin +(`q''_b`, Eq. 7), then applies gamma-distribution unit-hydrograph routing to +produce `q_b` — a single time series per gage basin, architecturally +incapable of native per-unit resolution. Critically, the paper's own +Muskingum-Cunge arm (δHBV2.0δMC) is fed `q_m` — the **pre-aggregation, +pre-UH-routing** per-unit runoff — not `q_b`; that is what our ddrs MC +routing should also receive. + +Tested the naive version of the hypothesis directly against the store: for +a 578-reach gauge subgraph (USGS 01184000), pulled `Qr` for 20 COMIDs +spanning headwater to near-outlet from both `daily_dhbv_aorc2f_...ic` +(distributed) and `daily_dhbv2_...ic` (lumped) via icechunk/zarr. **Result: +zero bit-identical pairs in either store** (0/190 pairwise comparisons, +1e-6 tolerance) — every COMID has a genuinely distinct time series in both +stores. Since `q_b` is architecturally one value per gage basin, distinct +per-COMID values rule out the simplest failure mode (the whole-basin +UH-routed `q_b` naively replicated across every upstream unit); this store +is very likely storing something closer to genuine per-unit `q_m`, not a +disaggregated `q_b`. A secondary check (median lag-1 day autocorrelation, +day-to-day-difference volatility, 4-year window, same 20 COMIDs) found +distributed *somewhat* smoother than lumped (day-to-day volatility 0.61 vs +0.98, autocorrelation 0.74 vs 0.77 — comparable) — a weak signal consistent +with, but far from proof of, *some* additional smoothing/partial routing in +the distributed store; a single-basin, two-metric comparison is not +statistically powered to decide this. **Net effect: the strong form of the +double-routing hypothesis (replicated basin-outlet flow) is refuted; a +weaker form (some partial pre-smoothing) remains an open, inconclusive +question.** Given the store's provenance is an internal `AORC2F`/`CONUS2717` +checkpoint from the same model family as the cited paper (not named in the +paper itself), the fastest way to close this out is to check the actual +export/postprocessing script that generated `daily_dhbv_aorc2f_merit_unit_catchments.ic` +for whether it exports `q_m` directly or something derived from `q_b`, rather +than inferring it purely from the stored values. Not yet tried: repeat this +comparison across multiple basins of varying size, and cross-correlate +against USGS peak timing directly. **Other gaps identified in a 2026-07-21 adversarial review of this campaign** (`/tmp/experiment-handoff-aorc2f-lstm-routing.md`): no