diff --git a/crates/stella-pipeline/src/management_prompt.rs b/crates/stella-pipeline/src/management_prompt.rs index 3998c26b3..fac0ef6af 100644 --- a/crates/stella-pipeline/src/management_prompt.rs +++ b/crates/stella-pipeline/src/management_prompt.rs @@ -21,6 +21,18 @@ //! structurally *never*, and the settings-supplied `agents..prompt` //! override joins the same system prefix, so a tuned role clears the minimum //! sooner, not later. +//! +//! Measured (#1786, 2026-08, estimator scale): no fixed block clears the +//! Anthropic minimum on its own — the verdict instructions estimate ~520 +//! tokens (the golden fixture records the exact figure), guidance and triage +//! less, and the witness author's system prompt ~620. So for the RAW calls +//! (triage/verdict/guidance) the split's cache win is real only with an +//! `agents..prompt` override padding the prefix past the minimum; +//! stability across calls is what the split guarantees, not a hit. The +//! witness author is different: it runs an ENGINE turn, where the provider +//! prefix is system prompt + conversation, which crosses the minimum within +//! the first tool round-trip — there the split converts the whole fixed +//! block from per-step re-billing into cached prefix from step two on. use stella_protocol::CompletionMessage; diff --git a/crates/stella-pipeline/src/pipeline.rs b/crates/stella-pipeline/src/pipeline.rs index 5492dc0b4..e20bdb05e 100644 --- a/crates/stella-pipeline/src/pipeline.rs +++ b/crates/stella-pipeline/src/pipeline.rs @@ -695,6 +695,12 @@ struct CandidateState { /// the audit runs and wherever it cannot be made — the two are the same /// claim, which is none. diff_coverage: DiffCoverage, + /// How the authored witness's arming failure presented (#1790): the + /// airlock's symptom class of the failing baseline run, recorded only + /// when it was a build failure — a flip armed by a compile error is + /// legitimate for a missing-API goal but weaker evidence than an + /// assertion failure, and the verifier deserves to see which it was. + witness_baseline_symptom: Option<&'static str>, revisions: u32, /// How many of those revisions were spent asking for corroboration of a /// standalone verifier pass rather than fixing a failure (#1295). Capped at @@ -2026,6 +2032,7 @@ impl<'a> Pipeline<'a> { revisions: 0, evidence_demands: 0, witness_paths: Vec::new(), + witness_baseline_symptom: None, failures: Vec::new(), last_verdict: None, last_verdict_diff: None, diff --git a/crates/stella-pipeline/src/pipeline/evidence.rs b/crates/stella-pipeline/src/pipeline/evidence.rs index d0b6b5b11..1cb03b66d 100644 --- a/crates/stella-pipeline/src/pipeline/evidence.rs +++ b/crates/stella-pipeline/src/pipeline/evidence.rs @@ -121,6 +121,15 @@ impl<'a> Pipeline<'a> { crate::replay::render_oracle_trace(&snapshot.oracle_trace) )); } + if let Some(symptom) = state.witness_baseline_symptom { + // #1790: the flip's arming failure never ran a test. Legitimate + // for a missing-API goal, indistinguishable by class from + // two-tree environment drift — so the verifier weighs it rather + // than the pipeline silently crediting or refusing it. + evidence_summary.push_str(&format!( + "; witness_baseline={symptom} (the arming failure ran no test)" + )); + } if snapshot.witness_intact == Some(true) { // #864: the tamper-exclusion result, stated. A tampered witness // never reaches a verifier, so what the verifier learns here is diff --git a/crates/stella-pipeline/src/pipeline/witness_stage.rs b/crates/stella-pipeline/src/pipeline/witness_stage.rs index b21350106..0b2546b12 100644 --- a/crates/stella-pipeline/src/pipeline/witness_stage.rs +++ b/crates/stella-pipeline/src/pipeline/witness_stage.rs @@ -57,6 +57,18 @@ fn apply_role_shaping(mut config: EngineConfig, overrides: &RoleCallOverrides) - config } +/// The recorded symptom of the authored witness's arming failure (#1790): +/// `Some("build_failure")` when the failing baseline never ran a test — +/// classified by the airlock's lexical rules over the run's own output — +/// and `None` for every failure shape that reached an assertion. Pure so +/// the classification is testable apart from the stage. +fn witness_baseline_symptom(baseline_output: &str) -> Option<&'static str> { + match crate::witness::airlock::SymptomClass::classify(baseline_output) { + crate::witness::airlock::SymptomClass::BuildFailure => Some("build_failure"), + _ => None, + } +} + /// Candidate-bound hook execution: both the hook process and the engine's /// payload use the isolated root, never the session root. pub(super) struct BoundHookRunner<'a> { @@ -81,9 +93,11 @@ impl HookRunner for BoundHookRunner<'_> { /// /// `degradable` = the witness simply couldn't be AUTHORED (no test command, /// an unusable command, a test that proves nothing, the author engine getting -/// stuck). The task needs no witness to proceed, so the run degrades to a -/// bare worker turn rather than dying. NOT degradable = a resource limit -/// (budget) or an artifact-INTEGRITY violation (the author modified tracked +/// stuck, a budget stop mid-authoring — #1789: the worker's change is already +/// done, and the budget guard still gates every later paid call, so degrading +/// preserves the work without overspending). The task needs no witness to +/// proceed, so the run degrades to a bare worker turn rather than dying. NOT +/// degradable = an artifact-INTEGRITY violation (the author modified tracked /// files, produced a non-single-file / symlink artifact, or a runner/identity /// mismatch) — fail-closed decisions that surface the problem rather than /// silently completing unverified. @@ -264,9 +278,16 @@ impl<'a> Pipeline<'a> { let tracked_before = baseline.repo_status.tracked_fingerprints().await; let untracked_before = baseline.repo_status.untracked_fingerprints().await; let structure = self.repo.structure_summary().await; - let baseline_workspace = baseline - .workspace - .expect("witness authoring requires a pristine baseline workspace"); + // Degrade, never panic (#1789): this stage's whole contract is that a + // witness which cannot be produced must not cost the run — a caller + // wiring a surface without a workspace is exactly such a case, and + // the one stage designed to degrade must not be the one that panics. + let Some(baseline_workspace) = baseline.workspace else { + return Err(WitnessAbort::degradable( + "witness authoring requires a pristine baseline workspace and none was provided" + .to_string(), + )); + }; let witness_tools = baseline_workspace.witness_tools(); let mut engine = Engine::with_sleeper( author.provider, @@ -316,9 +337,21 @@ impl<'a> Pipeline<'a> { TurnOutcome::Aborted { reason, cost_usd, .. } => { - *spend.total += cost_usd; - if let Some(abort) = budget_abort(spend.budget.evaluate()) { - return Err(WitnessAbort::rejected(abort.reason)); + *total += cost_usd; + // A budget stop here is degradable too (#1789): the worker's + // change is already complete in the candidate, and discarding + // it because the SCAFFOLDING ran out of money threw away real + // work. Degrading cannot overspend — every later paid call + // still passes the same budget guard, which aborts the run at + // the next unaffordable call; what degrading buys is the + // deterministically-resolvable endings (a warranted waiver, + // an abstention) that need no further model spend at all. + if let Some(abort) = budget_abort(budget.evaluate()) { + return Err(WitnessAbort::degradable(format!( + "witness authoring stopped by the budget ({}); the executed change \ + stands unproven", + abort.reason + ))); } return Err(WitnessAbort::degradable(format!( "witness author turn aborted: {reason}" @@ -400,9 +433,15 @@ impl<'a> Pipeline<'a> { TurnOutcome::Aborted { reason, cost_usd, .. } => { - *spend.total += cost_usd; - if let Some(abort) = budget_abort(spend.budget.evaluate()) { - return Err(WitnessAbort::rejected(abort.reason)); + *total += cost_usd; + // Degradable for the same #1789 reason as the author + // turn's budget arm above. + if let Some(abort) = budget_abort(budget.evaluate()) { + return Err(WitnessAbort::degradable(format!( + "witness repair stopped by the budget ({}); the executed change \ + stands unproven", + abort.reason + ))); } return Err(WitnessAbort::degradable(format!( "witness repair turn aborted: {reason}" @@ -491,9 +530,15 @@ impl<'a> Pipeline<'a> { // candidate's work is untouched and already done, so it falls through // to the unauthored ladder rather than being discarded for want of // scaffolding. - candidate - .workspace - .expect("witness grafting requires the candidate workspace") + // Same #1789 posture as the baseline workspace above: an absent + // candidate workspace loses the witness, never the run. + let Some(candidate_workspace) = candidate.workspace else { + return Err(WitnessAbort::degradable( + "witness grafting requires the candidate workspace and none was provided" + .to_string(), + )); + }; + candidate_workspace .graft_witness(baseline_workspace.root(), path) .await .map_err(|error| WitnessAbort::degradable(error.to_string()))?; @@ -623,6 +668,23 @@ impl<'a> Pipeline<'a> { .insert(path.clone(), identity.fingerprint.clone()); } state.witness_paths = witness.files.keys().cloned().collect(); + // #1790: a flip armed by a failure that never ran a test (a compile + // error) is legitimate — a missing-API witness FAILS TO BUILD on the + // old code by design — but it is weaker evidence than an assertion, + // and it is also the shape two-tree environment drift produces. It + // is recorded, surfaced in the verifier's evidence, and warned; it + // is deliberately not refused (that would reject the most common + // Rust witness shape). + state.witness_baseline_symptom = witness_baseline_symptom(&witness.baseline_output); + if state.witness_baseline_symptom.is_some() { + self.warn( + "the witness's failing baseline was a build failure, not a test failure — \ + correct for a missing-API goal, but the flip proves compilation, and \ + environment drift between the authoring snapshot and the candidate can \ + produce the same shape" + .to_string(), + ); + } Ok(Some(witness)) } } @@ -631,6 +693,24 @@ impl<'a> Pipeline<'a> { mod tests { use super::*; + /// #1790's witness: a compile-error baseline is RECORDED, not refused — + /// refusal would reject the most common Rust witness shape (a missing-API + /// test fails to build on the old code by design), while silence hid the + /// same shape when environment drift produced it. + #[test] + fn a_build_failure_baseline_is_recorded_and_an_assertion_one_is_not() { + assert_eq!( + witness_baseline_symptom("error[E0425]: cannot find function `retry_delays`"), + Some("build_failure") + ); + assert_eq!( + witness_baseline_symptom( + "thread 'witness' panicked at 'assertion failed: `(left == right)`'" + ), + None + ); + } + /// #1785's witness: the verifier's request shaping reaches the witness /// engines field by field, `None` overrides leave the worker's values /// standing, and `prompt` has no channel here at all (it is not an diff --git a/crates/stella-pipeline/src/verify.rs b/crates/stella-pipeline/src/verify.rs index 0c38a41bd..b5fcc5785 100644 --- a/crates/stella-pipeline/src/verify.rs +++ b/crates/stella-pipeline/src/verify.rs @@ -933,16 +933,28 @@ pub fn bound_forwarded_reasoning(text: &str) -> String { /// The conservative heuristic verdict used when the *verifier model call itself* /// fails or its response is unparseable (L-E11: "a heuristic fallback verdict /// if the verifier call itself fails"). It never fabricates confidence: it -/// passes only when the touched tests were observed green, and otherwise -/// fails (so an unverifiable turn is revised rather than shipped). A verifier -/// outage therefore degrades to "trust green tests, distrust everything -/// else", never to a blanket pass. +/// passes only on positive deterministic evidence — an observed fail→pass +/// flip, or touched tests observed green — and otherwise fails, so a turn +/// with nothing deterministic behind it is revised rather than shipped. +/// +/// The flip counts here for the same reason `Unverifiable` abstains instead +/// of failing (#1788): a verifier OUTAGE is the absence of a checker, not a +/// refutation, and it must not outrank the strongest deterministic evidence +/// this crate has. Before this, a candidate whose flip was confirmed but +/// whose diff ran over budget (routing it to the model verifier) was driven +/// to `VerificationFailed` by a provider being down. With neither flip nor +/// green tests the fallback still fails closed: the escalation existed +/// because something was genuinely inconclusive, and a revision is the +/// honest next move. pub fn heuristic_fallback(inputs: &LadderInputs) -> Verdict { - let passed = inputs.touched_tests_passed == Some(true); - let reasoning = if passed { + let passed = inputs.flip_achieved || inputs.touched_tests_passed == Some(true); + let reasoning = if inputs.flip_achieved { + "verifier unavailable; heuristic fallback passed on the observed fail→pass flip".to_string() + } else if passed { "verifier unavailable; heuristic fallback passed on green touched tests".to_string() } else { - "verifier unavailable; heuristic fallback failed (touched tests not confirmed green)" + "verifier unavailable; heuristic fallback failed (no flip, touched tests not \ + confirmed green)" .to_string() }; Verdict { diff --git a/crates/stella-pipeline/src/verify/tests.rs b/crates/stella-pipeline/src/verify/tests.rs index 72542b302..042fd9e7e 100644 --- a/crates/stella-pipeline/src/verify/tests.rs +++ b/crates/stella-pipeline/src/verify/tests.rs @@ -703,9 +703,15 @@ fn heuristic_fallback_passes_only_on_confirmed_green_tests() { }); assert!(green.passed); + // #1788: the flip rescues the fallback. A verifier OUTAGE is the absence + // of a checker, not a refutation, and the confirmed fail→pass flip is + // the strongest deterministic evidence the crate has — a provider being + // down must not convert it into VerificationFailed. (This inverts the + // earlier "even a flip doesn't rescue" pin, which treated the checker's + // absence as the work's failure.) for tests in [Some(false), None] { let v = heuristic_fallback(&LadderInputs { - flip_achieved: true, // even a flip doesn't rescue an unconfirmed suite + flip_achieved: true, touched_tests_passed: tests, diff_lines: 0, diff_budget: 100, @@ -714,8 +720,25 @@ fn heuristic_fallback_passes_only_on_confirmed_green_tests() { mutating_actions: 1, ..Default::default() }); - assert!(!v.passed, "unconfirmed tests must fall back to FAIL"); + assert!(v.passed, "a confirmed flip must survive a verifier outage"); } + // With NOTHING deterministic positive, the fallback still fails closed: + // the escalation existed because something was inconclusive, and a + // revision is the honest next move. + let v = heuristic_fallback(&LadderInputs { + flip_achieved: false, + touched_tests_passed: None, + diff_lines: 5, + diff_budget: 100, + diff_available: true, + file_change_events: 1, + mutating_actions: 1, + ..Default::default() + }); + assert!( + !v.passed, + "no positive evidence must still fall back to FAIL" + ); } #[test]