From 4d9948612e8e0d4e481d5e381b6bda75adf3bcb1 Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 03:17:01 -0700 Subject: [PATCH 1/3] feat(stella-pipeline): enforce and record verifier independence for the verdict call - require_independent_verifier refuses before spend when the verdict would resolve to the worker's own model (parity with the witness gate) - every model verdict's LadderSnapshot now states grader independence as a structured fact (verifier_independent), stamped at the call seam - settings key pipeline_require_independent_verifier wires the opt-in --- crates/stella-cli/src/agent/engine.rs | 4 + crates/stella-cli/src/settings.rs | 21 ++ crates/stella-cli/src/trace.rs | 1 + crates/stella-pipeline/src/pipeline.rs | 42 +++- .../stella-pipeline/src/pipeline/run_error.rs | 13 + crates/stella-pipeline/src/pipeline/tests.rs | 8 +- .../src/pipeline/tests/terminal_outcomes.rs | 2 +- .../pipeline/tests/verifier_independence.rs | 227 ++++++++++++++++++ .../src/pipeline/tests/witness_isolation.rs | 4 +- .../src/pipeline/verifier_stage.rs | 12 +- .../src/pipeline/verify_probes.rs | 4 + crates/stella-pipeline/src/replay.rs | 12 + crates/stella-pipeline/src/reward/tests.rs | 1 + crates/stella-pipeline/src/verify.rs | 14 ++ crates/stella-pipeline/src/verify/tests.rs | 1 + crates/stella-protocol/src/event.rs | 1 + crates/stella-protocol/src/ladder.rs | 54 ++++- crates/stella-protocol/tests/wire_contract.rs | 1 + 18 files changed, 411 insertions(+), 11 deletions(-) create mode 100644 crates/stella-pipeline/src/pipeline/tests/verifier_independence.rs diff --git a/crates/stella-cli/src/agent/engine.rs b/crates/stella-cli/src/agent/engine.rs index e40b96cae..710da5cbf 100644 --- a/crates/stella-cli/src/agent/engine.rs +++ b/crates/stella-cli/src/agent/engine.rs @@ -386,6 +386,10 @@ pub(crate) fn apply_pipeline_tuning(cfg: &Config, mut config: PipelineConfig) -> // #1291: the strict reading of an UNMEASURABLE overlap. A measured // non-overlap withholds the deterministic credit whatever this says. config.require_diff_coverage = engine.pipeline_require_diff_coverage_on(); + // #1795: refuse a self-graded verdict where the operator said so. Absent + // is off — the single-provider seat keeps its soft path, and the verdict + // records grader independence on its snapshot either way. + config.require_independent_verifier = engine.pipeline_require_independent_verifier_on(); if let Some(candidates) = engine.pipeline_candidates { // Stored as written, including `0`. `PipelineConfig::candidate_count` // floors at 1, so a zero reads as single-shot rather than as "run diff --git a/crates/stella-cli/src/settings.rs b/crates/stella-cli/src/settings.rs index c78a49f4b..a675036e9 100644 --- a/crates/stella-cli/src/settings.rs +++ b/crates/stella-cli/src/settings.rs @@ -471,6 +471,19 @@ pub struct AgentEngineConfig { /// which is what makes the two arms one binary and one posture key apart. #[serde(default, skip_serializing_if = "Option::is_none")] pub pipeline_verifier_evidence_demand: Option, + /// Refuse a run whose VERDICT call would resolve to the worker's own + /// model (`stella_pipeline::PipelineConfig::require_independent_verifier`, + /// #1795). Absent is off. + /// + /// Off is the right default for a single-provider BYOK seat, where the + /// verifier legitimately rides the worker and the verdict records that + /// fact on its ladder snapshot instead. Turn it on where the posture + /// claims an independent reviewer — a benchmark arm, a policy that treats + /// a self-graded PASS as no verdict at all — and a run that cannot honour + /// the claim should refuse before spending rather than produce a number + /// the configuration misdescribes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pipeline_require_independent_verifier: Option, /// Seconds of provider silence that end a single generation /// (`stella_core::EngineConfig::model_timeout`). Absent keeps the engine's /// own default, which is what every run used before this key existed. @@ -729,6 +742,7 @@ impl AgentEngineConfig { take!(pipeline_candidates); take!(pipeline_verifier_evidence_demand); take!(pipeline_require_diff_coverage); + take!(pipeline_require_independent_verifier); take!(model_timeout_secs); take!(compaction_budget_tokens); take!(tool_result_horizon_steps); @@ -836,6 +850,13 @@ impl AgentEngineConfig { .is_some_and(Toggle::is_on) } + /// Whether a verdict that would be graded by the worker's own model + /// refuses the run instead (#1795). Absent is off. + pub fn pipeline_require_independent_verifier_on(&self) -> bool { + self.pipeline_require_independent_verifier + .is_some_and(Toggle::is_on) + } + /// Whether interactive surfaces gate writes on per-hunk approval (#1265). pub fn hunk_review_on(&self) -> bool { self.hunk_review.is_some_and(Toggle::is_on) diff --git a/crates/stella-cli/src/trace.rs b/crates/stella-cli/src/trace.rs index ebf18c489..1361ba588 100644 --- a/crates/stella-cli/src/trace.rs +++ b/crates/stella-cli/src/trace.rs @@ -878,6 +878,7 @@ mod tests { witness_intact: None, witness_mutation: None, diff_coverage: None, + verifier_independent: None, } } } diff --git a/crates/stella-pipeline/src/pipeline.rs b/crates/stella-pipeline/src/pipeline.rs index 01e4dea13..c62b8d49d 100644 --- a/crates/stella-pipeline/src/pipeline.rs +++ b/crates/stella-pipeline/src/pipeline.rs @@ -378,6 +378,18 @@ pub struct PipelineConfig { /// number at all: the arm's own digest describes a configuration the run /// did not have (#1147). pub require_independent_witness: bool, + /// Refuse the run when the VERDICT call would resolve to the worker's own + /// model (#1795) — the "independent code reviewer" grading the code it + /// wrote — instead of proceeding with the once-per-run prose caveat. + /// + /// Off by default for the same reason its witness sibling above is: a + /// single-provider BYOK seat is the common case and must keep working. + /// On or off, the verdict's ladder snapshot records grader independence + /// as a structured fact (`LadderSnapshot::verifier_independent`), so a + /// stored verdict states it without the transcript. Checked before spend, + /// like the witness gate: a refusal after the trajectory is bought is a + /// trajectory the caller must throw away. + pub require_independent_verifier: bool, /// Best-of-N (L-E7). `None` or `Some(1)` is single-shot (the default); /// `Some(n)` generates n candidate executions — each in an isolated /// snapshot of the current tree state when a @@ -449,6 +461,7 @@ impl Default for PipelineConfig { // evidence-demand-1295/README.md` for the measurement. verifier_evidence_demand: true, require_independent_witness: false, + require_independent_verifier: false, candidates: None, candidate_concurrency: None, create_worktrees: crate::ports::WorktreePolicy::default(), @@ -930,6 +943,19 @@ impl<'a> Pipeline<'a> { total_cost, )); } + // Same probe, second consequence (#1795): the VERDICT grader must be + // independent too when the caller says so. The probe compares the + // worker's and verifier's resolved model refs, which is exactly + // "would the verdict resolve to the worker's model". + if self.config.require_independent_verifier + && let WitnessAuthorIndependence::Unavailable(reason) = + self.witness_author_independence() + { + return Err(PipelineRunError::new( + PipelineError::VerifierNotIndependent(reason), + total_cost, + )); + } if messages.is_empty() { messages.push(CompletionMessage::system(DEFAULT_SYSTEM_PROMPT)); } @@ -2790,7 +2816,11 @@ impl<'a> Pipeline<'a> { state.last_verdict_diff = Some(stripped.diff.clone()); } let mut evidence = model_verdict_evidence(&verdict); - evidence.ladder = Some(Box::new(snapshot.with_rung(verdict.rung()))); + evidence.ladder = Some(Box::new( + snapshot + .with_rung(verdict.rung()) + .with_verifier_independence(verdict.verifier_independent), + )); self.emit(AgentEvent::Verdict { passed: verdict.passed, evidence: evidence.clone(), @@ -3220,12 +3250,18 @@ impl<'a> Pipeline<'a> { Ok(verifier) if verifier.model_ref != worker.model_ref => { WitnessAuthorIndependence::Independent } + // Role-neutral wording on purpose (#1795): the same finding is + // framed by two different refusals (witness author, verdict + // grader) and one degradation notice, and each supplies its own + // role — a reason that named one would misname the others. Ok(_) => WitnessAuthorIndependence::Unavailable(format!( - "no author independent of the worker (verifier and worker both resolved to `{}`)", + "no model independent of the worker resolves (verifier and worker both \ + resolved to `{}`)", worker.model_ref )), Err(_) => WitnessAuthorIndependence::Unavailable( - "no author independent of the worker (the verifier role is unresolvable)" + "no model independent of the worker resolves (the verifier role is \ + unresolvable)" .to_string(), ), } diff --git a/crates/stella-pipeline/src/pipeline/run_error.rs b/crates/stella-pipeline/src/pipeline/run_error.rs index 5a66beb36..f98bbbcdf 100644 --- a/crates/stella-pipeline/src/pipeline/run_error.rs +++ b/crates/stella-pipeline/src/pipeline/run_error.rs @@ -32,6 +32,19 @@ pub enum PipelineError { "an independent witness author was required but {0} — refusing rather than running as the single-model arm under a configuration that claims otherwise" )] WitnessAuthorUnavailable(String), + /// [`crate::PipelineConfig::require_independent_verifier`] is on and the + /// verdict call would resolve to the worker's own model — or to no model + /// at all (#1795). + /// + /// Same shape and same before-spend placement as the witness refusal + /// above, for the same caller: one that has published the claim that an + /// independent reviewer grades the work. The ordinary posture keeps the + /// soft path — the verdict runs self-graded, records the fact on its + /// ladder snapshot, and the router's caveat says so in prose. + #[error( + "an independent verifier was required for the verdict but {0} — refusing before spend rather than letting the worker grade its own work under a configuration that claims otherwise" + )] + VerifierNotIndependent(String), /// A required role (worker) could not be resolved at all. #[error(transparent)] Routing(#[from] RouterError), diff --git a/crates/stella-pipeline/src/pipeline/tests.rs b/crates/stella-pipeline/src/pipeline/tests.rs index f3c58df35..aa5f2c270 100644 --- a/crates/stella-pipeline/src/pipeline/tests.rs +++ b/crates/stella-pipeline/src/pipeline/tests.rs @@ -1888,7 +1888,7 @@ async fn single_model_config_degrades_to_unauthored_witness_instead_of_aborting( matches!( event, AgentEvent::Error { message, retryable: true } - if message.contains("no author independent of the worker") + if message.contains("no model independent of the worker") ) }); assert!(warned, "the degradation is announced: {events:?}"); @@ -1897,7 +1897,7 @@ async fn single_model_config_degrades_to_unauthored_witness_instead_of_aborting( event, AgentEvent::Proof { step: stella_protocol::ProofStep::WitnessUnavailable { reason } - } if reason.contains("no author independent of the worker") + } if reason.contains("no model independent of the worker") ) }); assert!( @@ -2382,6 +2382,10 @@ mod verification_hardening; /// declining to ask where no tracked command could ever answer. A child /// module, so it reaches the scripted ports above via `super::*`. mod verifier_evidence_demand; +/// Verifier != worker for the verdict call (#1795): the opt-in refusal and +/// the structured grader-independence fact on the snapshot. A child module, +/// so it reaches the scripted ports above via `super::*`. +mod verifier_independence; /// Proportionate verification: changes with nothing to prove complete with a /// stated reason rather than escalating. A child module, so it reaches the /// scripted ports above via `super::*`. diff --git a/crates/stella-pipeline/src/pipeline/tests/terminal_outcomes.rs b/crates/stella-pipeline/src/pipeline/tests/terminal_outcomes.rs index 104c13c12..e983fc904 100644 --- a/crates/stella-pipeline/src/pipeline/tests/terminal_outcomes.rs +++ b/crates/stella-pipeline/src/pipeline/tests/terminal_outcomes.rs @@ -357,7 +357,7 @@ async fn unavailable_independent_witness_degrades_instead_of_aborting() { events.iter().any(|event| matches!( event, AgentEvent::Error { message, retryable: true } - if message.contains("no author independent of the worker") + if message.contains("no model independent of the worker") )), "the degradation is announced once: {events:?}" ); diff --git a/crates/stella-pipeline/src/pipeline/tests/verifier_independence.rs b/crates/stella-pipeline/src/pipeline/tests/verifier_independence.rs new file mode 100644 index 000000000..859617fb7 --- /dev/null +++ b/crates/stella-pipeline/src/pipeline/tests/verifier_independence.rs @@ -0,0 +1,227 @@ +//! #1795: nothing enforced verifier != worker for the VERDICT call. +//! +//! The witness author has a hard independence gate; the verdict call had +//! none — with one configured provider the "independent code reviewer" is +//! the model that wrote the code, and its PASS ends the run behind a +//! once-per-run prose caveat. These scenarios pin the two halves of the fix: +//! the opt-in refusal (`require_independent_verifier`, parity with the +//! witness gate, before any paid call), and the structured fact — every +//! model verdict's ladder snapshot states whether its grader was independent +//! (`LadderSnapshot::verifier_independent`), so a stored verdict says so +//! without the transcript. + +use super::*; + +/// A router whose worker and verifier pins resolve to `worker_model` and +/// `verifier_model` — the two-line variable every scenario here turns on. +fn pinned_router(worker_model: &str, verifier_model: &str) -> Router { + let mut roles = RoleTable::new(); + roles.pin(Role::Worker, ModelRef::new("scripted", worker_model)); + roles.pin(Role::Verifier, ModelRef::new("scripted", verifier_model)); + Router::new( + roles, + vec![ProviderProfile::new( + "scripted", + ModelRef::new("scripted", "worker"), + ModelRef::new("scripted", "triage"), + ModelRef::new("scripted", "verifier"), + )], + CircuitBreaker::new(Box::new(ZeroClock)), + ) +} + +/// One scripted run against `router`, returning the outcome (or refusal) and +/// the drained event stream. +macro_rules! independence_scenario { + ($provider:expr, $runner:expr, $router:expr, $config:expr) => {{ + let provider = $provider; + let resolver = OneProvider(&provider); + let runner = $runner; + let repo_status = SeqRepoStatus::new(vec![vec![]]); + let tools = EmptyTools; + let recall = NoContextRecall; + let repo = NoRepoStructure; + let approvals = AutoApproveGate; + let sleeper = NoopSleeper; + let router = $router; + let (tx, mut rx) = mpsc::unbounded_channel(); + let pipeline = Pipeline::new( + PipelinePorts { + router: &router, + providers: &resolver, + tools: &tools, + recall: &recall, + repo: &repo, + repo_status: &repo_status, + touches: &NoFileTouches, + diagnostics: &runner, + tests: &runner, + lint: None, + mutation: None, + coverage: None, + approvals: &approvals, + sleeper: &sleeper, + hooks: None, + candidate_workspaces: None, + mcp_prefetch: None, + steering: None, + }, + tx, + $config, + ); + let mut messages = vec![CompletionMessage::system("sys")]; + let mut budget = BudgetGuard::new(BudgetMode::Off, None, None); + let outcome = pipeline + .run("Fix the failing test", &mut messages, &mut budget) + .await; + (outcome, drain(&mut rx)) + }}; +} + +/// The verdict-path script every recording scenario shares: red baseline, +/// then an inconclusive candidate run, so the ladder escalates to the model +/// verifier — whose FAIL ends the run at `max_revisions: 0` with the stamped +/// snapshot as the run's verdict. +fn verdict_config() -> PipelineConfig { + PipelineConfig { + test_command: Some("cargo test -p x".into()), + diff_diagnostic: Some(DiagnosticInvocation::GitDiff), + // No repair round: the first model verdict is the outcome under test, + // and a revise turn after it would only lengthen the script. + max_revisions: 0, + distress_guidance: false, + ..PipelineConfig::default() + } +} + +/// The refusal witness (parity with +/// `requiring_an_independent_witness_refuses_before_spending_anything`): with +/// the flag set and the verdict resolving to the worker's own model, the run +/// refuses BEFORE any paid call — the provider is scripted with no responses, +/// so reaching one would panic rather than merely fail an assertion. +#[tokio::test] +async fn requiring_an_independent_verifier_refuses_before_spending_anything() { + let (outcome, events) = independence_scenario!( + ScriptedProvider::new(vec![]), + ScriptedRunner::new(vec![], ""), + pinned_router("same-model", "same-model"), + PipelineConfig { + require_independent_verifier: true, + ..PipelineConfig::default() + } + ); + + let error = outcome.expect_err("a self-grading verdict under the flag must refuse"); + match &error.cause { + PipelineError::VerifierNotIndependent(reason) => assert!( + reason.contains("no model independent of the worker"), + "the refusal must carry the reason: {reason}" + ), + other => panic!("expected VerifierNotIndependent, got {other:?}"), + } + assert_eq!( + error.total_cost_usd, 0.0, + "the refusal exists so the caller never buys a trajectory it must discard" + ); + assert!( + stages(&events).is_empty(), + "no stage may open before the refusal" + ); +} + +/// Default off: a single-key BYOK seat must keep working, exactly as the +/// witness sibling's opt-in test pins for its flag. +#[tokio::test] +async fn the_verifier_independence_requirement_is_opt_in() { + assert!( + !PipelineConfig::default().require_independent_verifier, + "a single-provider setup legitimately grades with the worker's model; \ + the verdict records the fact instead (see the snapshot scenarios below)" + ); +} + +/// A distinct verifier satisfies the requirement, so the flag is inert on the +/// arm it protects — the treatment arm must RUN, not merely fail loudly. +#[tokio::test] +async fn requiring_an_independent_verifier_lets_a_distinct_verifier_through() { + let (outcome, _events) = independence_scenario!( + ScriptedProvider::new(vec![ + text_result("single"), + text_result("done"), + text_result("FAIL — the change misses the empty case"), + ]), + ScriptedRunner::scripted( + vec![TestScript::Fail, TestScript::TimeOut], + "@@ -1 +1 @@\n-old\n+new" + ), + pinned_router("worker-model", "reviewer-model"), + PipelineConfig { + require_independent_verifier: true, + ..verdict_config() + } + ); + + let outcome = outcome.expect("an independent verifier satisfies the requirement"); + assert!( + matches!(outcome.status, PipelineStatus::VerificationFailed { .. }), + "the run proceeded to a real verdict: {:?}", + outcome.status + ); +} + +/// The recording witness, self-graded side: with worker and verifier resolving +/// to one model (the single-provider seat), the model verdict's snapshot +/// states `verifier_independent: false` — a structured fact a reader of the +/// STORED verdict can see, unlike the prose caveat that scrolls away. +#[tokio::test] +async fn a_self_graded_verdict_states_the_fact_on_its_snapshot() { + let (outcome, _events) = independence_scenario!( + ScriptedProvider::new(vec![ + text_result("single"), + text_result("done"), + text_result("FAIL — the change misses the empty case"), + ]), + ScriptedRunner::scripted( + vec![TestScript::Fail, TestScript::TimeOut], + "@@ -1 +1 @@\n-old\n+new" + ), + pinned_router("same-model", "same-model"), + verdict_config() + ); + + let outcome = outcome.expect("the soft path proceeds and records"); + let verdict = outcome.verdict.expect("the model verdict is the outcome"); + let snapshot = verdict.ladder.as_deref().expect("the verdict carries its snapshot"); + assert_eq!(snapshot.rung, Some(stella_protocol::LadderRung::ModelVerdict)); + assert_eq!( + snapshot.verifier_independent, + Some(false), + "a worker grading its own work must be stated on the stored verdict, \ + not only in a scrolling warning" + ); +} + +/// The other polarity: a grader distinct from the worker is stated as +/// independent — `Some(true)`, distinguishable from both self-grading and +/// from a snapshot recorded before the fact existed. +#[tokio::test] +async fn an_independent_verdict_states_that_too() { + let (outcome, _events) = independence_scenario!( + ScriptedProvider::new(vec![ + text_result("single"), + text_result("done"), + text_result("FAIL — the change misses the empty case"), + ]), + ScriptedRunner::scripted( + vec![TestScript::Fail, TestScript::TimeOut], + "@@ -1 +1 @@\n-old\n+new" + ), + pinned_router("worker-model", "reviewer-model"), + verdict_config() + ); + + let outcome = outcome.expect("the run proceeds"); + let verdict = outcome.verdict.expect("the model verdict is the outcome"); + let snapshot = verdict.ladder.as_deref().expect("the verdict carries its snapshot"); + assert_eq!(snapshot.verifier_independent, Some(true)); +} diff --git a/crates/stella-pipeline/src/pipeline/tests/witness_isolation.rs b/crates/stella-pipeline/src/pipeline/tests/witness_isolation.rs index 16540d25a..e5589dd45 100644 --- a/crates/stella-pipeline/src/pipeline/tests/witness_isolation.rs +++ b/crates/stella-pipeline/src/pipeline/tests/witness_isolation.rs @@ -258,7 +258,7 @@ async fn authored_witness_degrades_when_verifier_is_worker() { events.iter().any(|event| matches!( event, AgentEvent::Error { message, retryable: true } - if message.contains("no author independent of the worker") + if message.contains("no model independent of the worker") )), "the degradation is announced once: {events:?}" ); @@ -1338,7 +1338,7 @@ async fn requiring_an_independent_witness_refuses_before_spending_anything() { match &error.cause { PipelineError::WitnessAuthorUnavailable(reason) => assert!( - reason.contains("no author independent of the worker"), + reason.contains("no model independent of the worker"), "the refusal must carry the reason, not just the verdict: {reason}" ), other => panic!("expected WitnessAuthorUnavailable, got {other:?}"), diff --git a/crates/stella-pipeline/src/pipeline/verifier_stage.rs b/crates/stella-pipeline/src/pipeline/verifier_stage.rs index c17942627..6ce223bc7 100644 --- a/crates/stella-pipeline/src/pipeline/verifier_stage.rs +++ b/crates/stella-pipeline/src/pipeline/verifier_stage.rs @@ -123,7 +123,17 @@ impl<'a> Pipeline<'a> { .await { Ok(result) => match parse_verifier_response(&result.text) { - Some(verdict) => Ok(verdict), + Some(mut verdict) => { + // The grader-independence fact (#1795), stated where the + // resolution that graded is in hand. `None` when the + // worker itself will not resolve — nothing to compare — + // never a guess in either direction. + verdict.verifier_independent = self + .resolve_provider(Role::Worker) + .ok() + .map(|worker| worker.model_ref != resolved.model_ref); + Ok(verdict) + } None => { self.warn_verifier_fallback( "the verifier's response did not follow the verdict protocol", diff --git a/crates/stella-pipeline/src/pipeline/verify_probes.rs b/crates/stella-pipeline/src/pipeline/verify_probes.rs index 59a5cc413..3d6f2c768 100644 --- a/crates/stella-pipeline/src/pipeline/verify_probes.rs +++ b/crates/stella-pipeline/src/pipeline/verify_probes.rs @@ -209,6 +209,10 @@ impl<'a> Pipeline<'a> { // and the absence of the field would be a third, unintended // meaning. diff_coverage: Some(inputs.diff_coverage.as_str().to_string()), + // Stamped by the model-verdict arm alone + // (`LadderSnapshot::with_verifier_independence`): on every other + // rung nothing graded, so independence is not a fact about them. + verifier_independent: None, } } diff --git a/crates/stella-pipeline/src/replay.rs b/crates/stella-pipeline/src/replay.rs index 5811da3b5..8bcb42d7e 100644 --- a/crates/stella-pipeline/src/replay.rs +++ b/crates/stella-pipeline/src/replay.rs @@ -435,6 +435,17 @@ pub fn verdict_provenance(evidence: &VerdictEvidence) -> Option { if let Some(coverage) = &snapshot.diff_coverage { out.push_str(&format!("; diff_coverage={coverage}")); } + // #1795: a provenance reader asking "who graded this?" gets the stored + // fact — `self-graded` is the finding this field exists to surface, and + // `independent` is stated too so its absence stays distinguishable from + // "recorded before the fact existed". + if let Some(independent) = snapshot.verifier_independent { + out.push_str(if independent { + "; grader=independent" + } else { + "; grader=self-graded (worker's own model)" + }); + } Some(out) } @@ -1330,6 +1341,7 @@ mod calibration_tests { witness_intact: None, witness_mutation: None, diff_coverage: None, + verifier_independent: None, }; AgentEvent::Verdict { passed, diff --git a/crates/stella-pipeline/src/reward/tests.rs b/crates/stella-pipeline/src/reward/tests.rs index 53f60501f..56dd48e29 100644 --- a/crates/stella-pipeline/src/reward/tests.rs +++ b/crates/stella-pipeline/src/reward/tests.rs @@ -39,6 +39,7 @@ fn snapshot(rung: Option) -> LadderSnapshot { witness_intact: None, witness_mutation: None, diff_coverage: None, + verifier_independent: None, } } diff --git a/crates/stella-pipeline/src/verify.rs b/crates/stella-pipeline/src/verify.rs index af3f6e674..0c38a41bd 100644 --- a/crates/stella-pipeline/src/verify.rs +++ b/crates/stella-pipeline/src/verify.rs @@ -800,6 +800,15 @@ pub struct Verdict { /// extraction discards the first and keeps the second, so the distinction /// has to survive as a field rather than as the wording of `reasoning`. pub heuristic: bool, + /// Whether the model that answered was independent of the worker (#1795): + /// `Some(false)` = the verdict call resolved to the worker's own model, + /// so this "independent code review" graded its own work. Stamped by the + /// call seam (`Pipeline::verifier`), which is the one place the actual + /// resolution is known — the parser and the heuristic fallback leave it + /// `None` (no model answered, or nobody compared). Carried onto the + /// verdict's `LadderSnapshot` so a stored verdict states the fact without + /// the transcript. + pub verifier_independent: Option, } impl Verdict { @@ -873,6 +882,7 @@ pub fn parse_verifier_response(text: &str) -> Option { passed: !negated, reasoning: text.trim().to_string(), heuristic: false, + verifier_independent: None, }); } "fail" | "failed" | "reject" | "rejected" if !negated => { @@ -880,6 +890,7 @@ pub fn parse_verifier_response(text: &str) -> Option { passed: false, reasoning: text.trim().to_string(), heuristic: false, + verifier_independent: None, }); } _ => {} @@ -938,6 +949,9 @@ pub fn heuristic_fallback(inputs: &LadderInputs) -> Verdict { passed, reasoning, heuristic: true, + // No model answered, so grader independence is not a fact about this + // verdict — absent, never false. + verifier_independent: None, } } diff --git a/crates/stella-pipeline/src/verify/tests.rs b/crates/stella-pipeline/src/verify/tests.rs index e27b73836..e85be1136 100644 --- a/crates/stella-pipeline/src/verify/tests.rs +++ b/crates/stella-pipeline/src/verify/tests.rs @@ -728,6 +728,7 @@ fn evidence_builders_tag_determinism_correctly() { passed: true, reasoning: "looks fine".into(), heuristic: false, + verifier_independent: None, }); assert!( !model.deterministic, diff --git a/crates/stella-protocol/src/event.rs b/crates/stella-protocol/src/event.rs index 1dc72aad6..70ce98ed0 100644 --- a/crates/stella-protocol/src/event.rs +++ b/crates/stella-protocol/src/event.rs @@ -2093,6 +2093,7 @@ mod tests { witness_intact: Some(true), witness_mutation: None, diff_coverage: Some("covered".into()), + verifier_independent: None, })), }, }; diff --git a/crates/stella-protocol/src/ladder.rs b/crates/stella-protocol/src/ladder.rs index 4d0d1d86a..dad540269 100644 --- a/crates/stella-protocol/src/ladder.rs +++ b/crates/stella-protocol/src/ladder.rs @@ -208,6 +208,21 @@ pub struct LadderSnapshot { /// `unmeasured`, never as either verdict. #[serde(default, skip_serializing_if = "Option::is_none")] pub diff_coverage: Option, + /// Whether the model that graded this verdict was independent of the + /// worker that produced the work (#1795): `Some(false)` is a self-graded + /// verdict — the verdict call resolved to the worker's own model — and + /// `Some(true)` a distinct grader. + /// + /// A structured fact rather than the once-per-run prose caveat, because + /// the caveat scrolls away while the verdict is stored: a reader of a + /// stored verdict must be able to see the grader was not independent + /// without the transcript. Absent when no model verdict was bought (the + /// deterministic, waived, and abstaining rungs — nothing graded, so + /// independence is not a fact about them), when the worker's own + /// resolution failed (nothing to compare against), and on snapshots + /// recorded before this existed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub verifier_independent: Option, } impl LadderSnapshot { @@ -221,6 +236,17 @@ impl LadderSnapshot { ..self.clone() } } + + /// This snapshot with the grader-independence fact set (#1795). Stamped + /// only on the model-verdict path — see [`Self::verifier_independent`] + /// for why the other rungs stay absent. + #[must_use] + pub fn with_verifier_independence(self, verifier_independent: Option) -> Self { + Self { + verifier_independent, + ..self + } + } } /// Which code state a `ProofStep::Oracle` observation was made against. @@ -271,20 +297,44 @@ mod tests { witness_intact: Some(true), witness_mutation: Some(true), diff_coverage: Some("covered".into()), + verifier_independent: None, } } /// Invariant #4: the snapshot round-trips byte-for-byte, with and without - /// a rung. + /// a rung, and with the grader-independence fact in either polarity. #[test] fn the_snapshot_round_trips() { - for value in [snapshot(), snapshot().with_rung(LadderRung::SubmitFast)] { + for value in [ + snapshot(), + snapshot().with_rung(LadderRung::SubmitFast), + snapshot() + .with_rung(LadderRung::ModelVerdict) + .with_verifier_independence(Some(false)), + ] { let json = serde_json::to_string(&value).unwrap(); let back: LadderSnapshot = serde_json::from_str(&json).unwrap(); assert_eq!(value, back); } } + /// The grader-independence fact is additive (#1795): a snapshot that never + /// stated it emits no key and parses as unknown — never as either verdict. + #[test] + fn verifier_independence_is_additive() { + let json = serde_json::to_string(&snapshot()).unwrap(); + assert!( + !json.contains("verifier_independent"), + "an unstated fact emits no key: {json}" + ); + let stamped = snapshot().with_verifier_independence(Some(false)); + let json = serde_json::to_string(&stamped).unwrap(); + assert!( + json.contains("\"verifier_independent\":false"), + "a self-graded verdict states it: {json}" + ); + } + /// `rung` is additive: a snapshot recorded before it existed still parses, /// and a snapshot without one does not emit the key. #[test] diff --git a/crates/stella-protocol/tests/wire_contract.rs b/crates/stella-protocol/tests/wire_contract.rs index 4ec1268a5..62e9e73f2 100644 --- a/crates/stella-protocol/tests/wire_contract.rs +++ b/crates/stella-protocol/tests/wire_contract.rs @@ -1161,6 +1161,7 @@ fn sample_events() -> Vec { witness_intact: Some(true), witness_mutation: Some(true), diff_coverage: Some("covered".into()), + verifier_independent: Some(false), })), }, }), From fbd98a574c14f2e9e71c21d166567caa1be542b3 Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 03:56:19 -0700 Subject: [PATCH 2/3] chore(stella-pipeline): split replay/verify test modules under the god-file limit; wire schema + baseline --- crates/stella-cli/src/settings/toml_config.rs | 4 + .../pipeline/tests/verifier_independence.rs | 15 ++- crates/stella-pipeline/src/replay.rs | 108 +---------------- .../src/replay/late_reconciliation_tests.rs | 109 ++++++++++++++++++ crates/stella-pipeline/src/verify/tests.rs | 66 +---------- .../src/verify/tests/witness_strip.rs | 70 +++++++++++ docs/wire/agentevent.d.ts | 16 +++ docs/wire/agentevent.schema.json | 7 ++ docs/wire/serveframe.d.ts | 16 +++ docs/wire/serveframe.schema.json | 7 ++ scripts/file-size-baseline.txt | 6 +- 11 files changed, 246 insertions(+), 178 deletions(-) create mode 100644 crates/stella-pipeline/src/replay/late_reconciliation_tests.rs create mode 100644 crates/stella-pipeline/src/verify/tests/witness_strip.rs diff --git a/crates/stella-cli/src/settings/toml_config.rs b/crates/stella-cli/src/settings/toml_config.rs index cf8b09608..93927bb75 100644 --- a/crates/stella-cli/src/settings/toml_config.rs +++ b/crates/stella-cli/src/settings/toml_config.rs @@ -193,6 +193,8 @@ pub struct AgentsSection { #[serde(default, skip_serializing_if = "Option::is_none")] pub pipeline_require_diff_coverage: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub pipeline_require_independent_verifier: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub model_timeout_secs: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub compaction_budget_tokens: Option, @@ -597,6 +599,7 @@ pub fn raise_agents(cfg: &AgentEngineConfig) -> (AgentsSection, ModelsSection) { pipeline_candidates: cfg.pipeline_candidates, pipeline_verifier_evidence_demand: cfg.pipeline_verifier_evidence_demand, pipeline_require_diff_coverage: cfg.pipeline_require_diff_coverage, + pipeline_require_independent_verifier: cfg.pipeline_require_independent_verifier, model_timeout_secs: cfg.model_timeout_secs, compaction_budget_tokens: cfg.compaction_budget_tokens, tool_result_horizon_steps: cfg.tool_result_horizon_steps, @@ -670,6 +673,7 @@ fn lower_agents(agents: AgentsSection, models: ModelsSection) -> Option AgentEvent { - AgentEvent::Verdict { - passed: true, - evidence: VerdictEvidence { - summary: String::new(), - deterministic: false, - evidence_refs: vec![], - ladder: None, - }, - } - } - - fn commit(sha: &str) -> AgentEvent { - AgentEvent::Commit { - sha: sha.into(), - message: "feat: the work".into(), - } - } - - fn pr(url: &str, ci: Option) -> AgentEvent { - AgentEvent::Pr { - url: url.into(), - status: PrStatus::Open, - number: Some(1), - ci, - } - } - - /// #1293 acceptance: a pass recorded after its session's last CI - /// observation used to be lost. It now leaves the fold carrying the - /// commit it covers, and a revert discovered later settles it. - #[test] - fn a_pass_after_the_last_ci_verdict_survives_its_session_and_a_revert_settles_it() { - let events = vec![ - verifier_pass(), - pr("https://example.test/pr/1", Some(CiStatus::Passing)), - // Everything below is after the session's last terminal verdict — - // the region that used to be unreachable. - verifier_pass(), - commit("abc123abc123"), - ]; - let (report, pending) = calibration_pending("sess-a", &events); - assert_eq!(report.verifier_passes, 2); - assert_eq!( - report.verifier_reconciled, 1, - "only the first pass had an in-stream verdict" - ); - assert_eq!(pending.len(), 1, "the trailing pass is carried out"); - assert_eq!(pending[0].session, "sess-a"); - assert_eq!(pending[0].commits, vec!["abc123abc123".to_string()]); - - let mut report = report; - let truth = GroundTruth::default().with_reverts(["abc123abc123".to_string()]); - assert_eq!(reconcile(&mut report, &pending, &truth), 1); - assert_eq!(report.verifier_reconciled, 2); - assert_eq!(report.verifier_false_positives, 1); - assert_eq!(report.verifier_reverted, 1); - assert!( - render_calibration(&report).contains("settled by a REVERT"), - "the render must distinguish a human's revert from a red CI run: {}", - render_calibration(&report) - ); - } - - /// The cross-session half: session A's trailing pass is settled by a - /// terminal CI verdict recorded in session B, for the PR they share. - #[test] - fn a_terminal_verdict_in_a_later_session_settles_an_earlier_ones_pass() { - let session_a = vec![verifier_pass(), pr("https://example.test/pr/4", None)]; - let session_b = vec![pr("https://example.test/pr/4", Some(CiStatus::Failing))]; - let (mut report, pending) = calibration_pending("sess-a", &session_a); - assert_eq!( - report.verifier_reconciled, 0, - "a PR with no terminal verdict reconciles nothing in its own stream" - ); - let truth = GroundTruth::default() - .with_stream(&session_a) - .with_stream(&session_b); - assert_eq!(reconcile(&mut report, &pending, &truth), 1); - assert_eq!(report.verifier_false_positives, 1); - assert_eq!( - report.verifier_reverted, 0, - "CI is not a revert, and the two must not merge into one number" - ); - } - - /// The unchanged contract: `calibration` still answers exactly what it - /// answered before, so a caller that only wants the in-stream reading is - /// untouched by any of this. - #[test] - fn the_in_stream_fold_is_unchanged() { - let events = vec![ - verifier_pass(), - pr("https://example.test/pr/2", Some(CiStatus::Failing)), - ]; - let report = calibration(&events); - assert_eq!(report.verifier_passes, 1); - assert_eq!(report.verifier_reconciled, 1); - assert_eq!(report.verifier_false_positives, 1); - assert_eq!(report.verifier_reverted, 0); - } -} +mod late_reconciliation_tests; diff --git a/crates/stella-pipeline/src/replay/late_reconciliation_tests.rs b/crates/stella-pipeline/src/replay/late_reconciliation_tests.rs new file mode 100644 index 000000000..648760c1d --- /dev/null +++ b/crates/stella-pipeline/src/replay/late_reconciliation_tests.rs @@ -0,0 +1,109 @@ +//! Late ground-truth reconciliation over a recorded stream — split out +//! of `replay.rs` to keep it under the file-size ratchet; a child +//! module, so the fold internals stay reachable via `super::*`. + +use super::ground_truth::{GroundTruth, reconcile}; +use super::*; +use stella_protocol::{CiStatus, PrStatus, VerdictEvidence}; + +fn verifier_pass() -> AgentEvent { + AgentEvent::Verdict { + passed: true, + evidence: VerdictEvidence { + summary: String::new(), + deterministic: false, + evidence_refs: vec![], + ladder: None, + }, + } +} + +fn commit(sha: &str) -> AgentEvent { + AgentEvent::Commit { + sha: sha.into(), + message: "feat: the work".into(), + } +} + +fn pr(url: &str, ci: Option) -> AgentEvent { + AgentEvent::Pr { + url: url.into(), + status: PrStatus::Open, + number: Some(1), + ci, + } +} + +/// #1293 acceptance: a pass recorded after its session's last CI +/// observation used to be lost. It now leaves the fold carrying the +/// commit it covers, and a revert discovered later settles it. +#[test] +fn a_pass_after_the_last_ci_verdict_survives_its_session_and_a_revert_settles_it() { + let events = vec![ + verifier_pass(), + pr("https://example.test/pr/1", Some(CiStatus::Passing)), + // Everything below is after the session's last terminal verdict — + // the region that used to be unreachable. + verifier_pass(), + commit("abc123abc123"), + ]; + let (report, pending) = calibration_pending("sess-a", &events); + assert_eq!(report.verifier_passes, 2); + assert_eq!( + report.verifier_reconciled, 1, + "only the first pass had an in-stream verdict" + ); + assert_eq!(pending.len(), 1, "the trailing pass is carried out"); + assert_eq!(pending[0].session, "sess-a"); + assert_eq!(pending[0].commits, vec!["abc123abc123".to_string()]); + + let mut report = report; + let truth = GroundTruth::default().with_reverts(["abc123abc123".to_string()]); + assert_eq!(reconcile(&mut report, &pending, &truth), 1); + assert_eq!(report.verifier_reconciled, 2); + assert_eq!(report.verifier_false_positives, 1); + assert_eq!(report.verifier_reverted, 1); + assert!( + render_calibration(&report).contains("settled by a REVERT"), + "the render must distinguish a human's revert from a red CI run: {}", + render_calibration(&report) + ); +} + +/// The cross-session half: session A's trailing pass is settled by a +/// terminal CI verdict recorded in session B, for the PR they share. +#[test] +fn a_terminal_verdict_in_a_later_session_settles_an_earlier_ones_pass() { + let session_a = vec![verifier_pass(), pr("https://example.test/pr/4", None)]; + let session_b = vec![pr("https://example.test/pr/4", Some(CiStatus::Failing))]; + let (mut report, pending) = calibration_pending("sess-a", &session_a); + assert_eq!( + report.verifier_reconciled, 0, + "a PR with no terminal verdict reconciles nothing in its own stream" + ); + let truth = GroundTruth::default() + .with_stream(&session_a) + .with_stream(&session_b); + assert_eq!(reconcile(&mut report, &pending, &truth), 1); + assert_eq!(report.verifier_false_positives, 1); + assert_eq!( + report.verifier_reverted, 0, + "CI is not a revert, and the two must not merge into one number" + ); +} + +/// The unchanged contract: `calibration` still answers exactly what it +/// answered before, so a caller that only wants the in-stream reading is +/// untouched by any of this. +#[test] +fn the_in_stream_fold_is_unchanged() { + let events = vec![ + verifier_pass(), + pr("https://example.test/pr/2", Some(CiStatus::Failing)), + ]; + let report = calibration(&events); + assert_eq!(report.verifier_passes, 1); + assert_eq!(report.verifier_reconciled, 1); + assert_eq!(report.verifier_false_positives, 1); + assert_eq!(report.verifier_reverted, 0); +} diff --git a/crates/stella-pipeline/src/verify/tests.rs b/crates/stella-pipeline/src/verify/tests.rs index e85be1136..72542b302 100644 --- a/crates/stella-pipeline/src/verify/tests.rs +++ b/crates/stella-pipeline/src/verify/tests.rs @@ -7,6 +7,7 @@ use proptest::prelude::*; mod parse; mod uncollected; +mod witness_strip; // FlipOracle transitions @@ -1357,68 +1358,3 @@ fn an_evidence_demand_is_bounded_on_every_axis() { } // strip_witness_hunks - -/// The witness is grafted into the candidate tree, so the working-tree diff -/// carries the verifier's own test. It must not ride into the verifier prompt -/// as "worker-authored data" — drop its chunks, return its paths for the -/// trusted evidence zone, and leave every worker chunk byte-intact. -#[test] -fn witness_chunks_are_stripped_and_named_never_billed() { - let diff = "diff --git a/src/lib.rs b/src/lib.rs\n\ - --- a/src/lib.rs\n\ - +++ b/src/lib.rs\n\ - @@ -1,2 +1,3 @@\n\ - fn f() {}\n\ - +fn g() {}\n\ - diff --git a/tests/witness_g.rs b/tests/witness_g.rs\n\ - --- /dev/null\n\ - +++ b/tests/witness_g.rs\n\ - @@ -0,0 +1,2 @@\n\ - +#[test]\n\ - +fn g_works() { g(); }\n"; - let stripped = strip_witness_hunks(diff, &["tests/witness_g.rs".to_string()]); - assert!( - stripped.diff.contains("+fn g() {}"), - "worker chunk survives:\n{}", - stripped.diff - ); - assert!( - !stripped.diff.contains("witness_g"), - "witness chunk is gone:\n{}", - stripped.diff - ); - assert_eq!(stripped.omitted, vec!["tests/witness_g.rs".to_string()]); -} - -/// No witness, no change: the common test-command path must pass the diff -/// through byte-identically. -#[test] -fn stripping_with_no_witness_paths_is_the_identity() { - let diff = "diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1 +1 @@\n+x\n"; - let stripped = strip_witness_hunks(diff, &[]); - assert_eq!(stripped.diff, diff); - assert!(stripped.omitted.is_empty()); -} - -/// Preamble lines before the first `diff --git` (the honest-diff banner, the -/// untracked-change notes) belong to no file and always survive; and an added -/// content line that happens to start with `+++ ` cannot hijack the chunk's -/// path because only the FIRST `+++ ` per chunk is read. -#[test] -fn preamble_survives_and_content_cannot_hijack_the_chunk_path() { - let diff = "3 files changed but the probe saw an empty diff\n\ - diff --git a/src/a.rs b/src/a.rs\n\ - --- a/src/a.rs\n\ - +++ b/src/a.rs\n\ - @@ -1 +1,2 @@\n\ - +++ b/tests/witness_g.rs\n\ - +real added line\n"; - let stripped = strip_witness_hunks(diff, &["tests/witness_g.rs".to_string()]); - assert!(stripped.diff.contains("empty diff"), "preamble survives"); - assert!( - stripped.diff.contains("+real added line"), - "the chunk keyed on its real header (src/a.rs), not on forged content:\n{}", - stripped.diff - ); - assert!(stripped.omitted.is_empty()); -} diff --git a/crates/stella-pipeline/src/verify/tests/witness_strip.rs b/crates/stella-pipeline/src/verify/tests/witness_strip.rs new file mode 100644 index 000000000..2ed6f0e58 --- /dev/null +++ b/crates/stella-pipeline/src/verify/tests/witness_strip.rs @@ -0,0 +1,70 @@ +//! Witness-hunk stripping for verifier-facing diff renders — split out +//! of `verify/tests.rs` to keep it under the file-size ratchet; a child +//! module, so the strip internals stay reachable via `super::*`. + +use super::*; + +/// The witness is grafted into the candidate tree, so the working-tree diff +/// carries the verifier's own test. It must not ride into the verifier prompt +/// as "worker-authored data" — drop its chunks, return its paths for the +/// trusted evidence zone, and leave every worker chunk byte-intact. +#[test] +fn witness_chunks_are_stripped_and_named_never_billed() { + let diff = "diff --git a/src/lib.rs b/src/lib.rs\n\ + --- a/src/lib.rs\n\ + +++ b/src/lib.rs\n\ + @@ -1,2 +1,3 @@\n\ + fn f() {}\n\ + +fn g() {}\n\ + diff --git a/tests/witness_g.rs b/tests/witness_g.rs\n\ + --- /dev/null\n\ + +++ b/tests/witness_g.rs\n\ + @@ -0,0 +1,2 @@\n\ + +#[test]\n\ + +fn g_works() { g(); }\n"; + let stripped = strip_witness_hunks(diff, &["tests/witness_g.rs".to_string()]); + assert!( + stripped.diff.contains("+fn g() {}"), + "worker chunk survives:\n{}", + stripped.diff + ); + assert!( + !stripped.diff.contains("witness_g"), + "witness chunk is gone:\n{}", + stripped.diff + ); + assert_eq!(stripped.omitted, vec!["tests/witness_g.rs".to_string()]); +} + +/// No witness, no change: the common test-command path must pass the diff +/// through byte-identically. +#[test] +fn stripping_with_no_witness_paths_is_the_identity() { + let diff = "diff --git a/a.rs b/a.rs\n--- a/a.rs\n+++ b/a.rs\n@@ -1 +1 @@\n+x\n"; + let stripped = strip_witness_hunks(diff, &[]); + assert_eq!(stripped.diff, diff); + assert!(stripped.omitted.is_empty()); +} + +/// Preamble lines before the first `diff --git` (the honest-diff banner, the +/// untracked-change notes) belong to no file and always survive; and an added +/// content line that happens to start with `+++ ` cannot hijack the chunk's +/// path because only the FIRST `+++ ` per chunk is read. +#[test] +fn preamble_survives_and_content_cannot_hijack_the_chunk_path() { + let diff = "3 files changed but the probe saw an empty diff\n\ + diff --git a/src/a.rs b/src/a.rs\n\ + --- a/src/a.rs\n\ + +++ b/src/a.rs\n\ + @@ -1 +1,2 @@\n\ + +++ b/tests/witness_g.rs\n\ + +real added line\n"; + let stripped = strip_witness_hunks(diff, &["tests/witness_g.rs".to_string()]); + assert!(stripped.diff.contains("empty diff"), "preamble survives"); + assert!( + stripped.diff.contains("+real added line"), + "the chunk keyed on its real header (src/a.rs), not on forged content:\n{}", + stripped.diff + ); + assert!(stripped.omitted.is_empty()); +} diff --git a/docs/wire/agentevent.d.ts b/docs/wire/agentevent.d.ts index 56bd6399b..7d4aa58ac 100644 --- a/docs/wire/agentevent.d.ts +++ b/docs/wire/agentevent.d.ts @@ -450,6 +450,22 @@ export interface LadderSnapshot { * A flip was observed but its confirmation re-run did not pass (#859). */ unstable_flip: boolean; + /** + * Whether the model that graded this verdict was independent of the + * worker that produced the work (#1795): `Some(false)` is a self-graded + * verdict — the verdict call resolved to the worker's own model — and + * `Some(true)` a distinct grader. + * + * A structured fact rather than the once-per-run prose caveat, because + * the caveat scrolls away while the verdict is stored: a reader of a + * stored verdict must be able to see the grader was not independent + * without the transcript. Absent when no model verdict was bought (the + * deterministic, waived, and abstaining rungs — nothing graded, so + * independence is not a fact about them), when the worker's own + * resolution failed (nothing to compare against), and on snapshots + * recorded before this existed. + */ + verifier_independent?: boolean | null; /** * The witness-tamper check's result: `None` when no witness was armed, * `Some(true)` when every witness artifact matched its pinned identity. diff --git a/docs/wire/agentevent.schema.json b/docs/wire/agentevent.schema.json index 675c29b7a..1c00c7f6b 100644 --- a/docs/wire/agentevent.schema.json +++ b/docs/wire/agentevent.schema.json @@ -597,6 +597,13 @@ "description": "A flip was observed but its confirmation re-run did not pass (#859).", "type": "boolean" }, + "verifier_independent": { + "description": "Whether the model that graded this verdict was independent of the\nworker that produced the work (#1795): `Some(false)` is a self-graded\nverdict — the verdict call resolved to the worker's own model — and\n`Some(true)` a distinct grader.\n\nA structured fact rather than the once-per-run prose caveat, because\nthe caveat scrolls away while the verdict is stored: a reader of a\nstored verdict must be able to see the grader was not independent\nwithout the transcript. Absent when no model verdict was bought (the\ndeterministic, waived, and abstaining rungs — nothing graded, so\nindependence is not a fact about them), when the worker's own\nresolution failed (nothing to compare against), and on snapshots\nrecorded before this existed.", + "type": [ + "boolean", + "null" + ] + }, "witness_intact": { "description": "The witness-tamper check's result: `None` when no witness was armed,\n`Some(true)` when every witness artifact matched its pinned identity.\n`Some(false)` never reaches a verdict — tampering aborts the\ncandidate — so its presence here is the *stated* proof the check ran.", "type": [ diff --git a/docs/wire/serveframe.d.ts b/docs/wire/serveframe.d.ts index 7d99bc2b6..09a919e55 100644 --- a/docs/wire/serveframe.d.ts +++ b/docs/wire/serveframe.d.ts @@ -1159,6 +1159,22 @@ export interface LadderSnapshot { * A flip was observed but its confirmation re-run did not pass (#859). */ unstable_flip: boolean; + /** + * Whether the model that graded this verdict was independent of the + * worker that produced the work (#1795): `Some(false)` is a self-graded + * verdict — the verdict call resolved to the worker's own model — and + * `Some(true)` a distinct grader. + * + * A structured fact rather than the once-per-run prose caveat, because + * the caveat scrolls away while the verdict is stored: a reader of a + * stored verdict must be able to see the grader was not independent + * without the transcript. Absent when no model verdict was bought (the + * deterministic, waived, and abstaining rungs — nothing graded, so + * independence is not a fact about them), when the worker's own + * resolution failed (nothing to compare against), and on snapshots + * recorded before this existed. + */ + verifier_independent?: boolean | null; /** * The witness-tamper check's result: `None` when no witness was armed, * `Some(true)` when every witness artifact matched its pinned identity. diff --git a/docs/wire/serveframe.schema.json b/docs/wire/serveframe.schema.json index 6b72cd05a..f9bffef46 100644 --- a/docs/wire/serveframe.schema.json +++ b/docs/wire/serveframe.schema.json @@ -2118,6 +2118,13 @@ "description": "A flip was observed but its confirmation re-run did not pass (#859).", "type": "boolean" }, + "verifier_independent": { + "description": "Whether the model that graded this verdict was independent of the\nworker that produced the work (#1795): `Some(false)` is a self-graded\nverdict — the verdict call resolved to the worker's own model — and\n`Some(true)` a distinct grader.\n\nA structured fact rather than the once-per-run prose caveat, because\nthe caveat scrolls away while the verdict is stored: a reader of a\nstored verdict must be able to see the grader was not independent\nwithout the transcript. Absent when no model verdict was bought (the\ndeterministic, waived, and abstaining rungs — nothing graded, so\nindependence is not a fact about them), when the worker's own\nresolution failed (nothing to compare against), and on snapshots\nrecorded before this existed.", + "type": [ + "boolean", + "null" + ] + }, "witness_intact": { "description": "The witness-tamper check's result: `None` when no witness was armed,\n`Some(true)` when every witness artifact matched its pinned identity.\n`Some(false)` never reaches a verdict — tampering aborts the\ncandidate — so its presence here is the *stated* proof the check ran.", "type": [ diff --git a/scripts/file-size-baseline.txt b/scripts/file-size-baseline.txt index b030d2807..19b6425a7 100644 --- a/scripts/file-size-baseline.txt +++ b/scripts/file-size-baseline.txt @@ -26,9 +26,9 @@ 2093 crates/stella-model/src/openai.rs 1565 crates/stella-model/src/zai.rs 1895 crates/stella-model/src/zai/tests.rs -3642 crates/stella-pipeline/src/pipeline.rs -2568 crates/stella-pipeline/src/pipeline/tests.rs -2962 crates/stella-protocol/src/event.rs +3616 crates/stella-pipeline/src/pipeline.rs +2572 crates/stella-pipeline/src/pipeline/tests.rs +2963 crates/stella-protocol/src/event.rs 1995 crates/stella-store/src/lib.rs 2268 crates/stella-store/src/tests.rs 1916 crates/stella-store/src/usage.rs From abe25423ca197733d1de82464f258b9af7cef87a Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 04:02:22 -0700 Subject: [PATCH 3/3] chore(stella-pipeline): regenerate wire schema and file-size baseline after rebase Closes #1795 --- scripts/file-size-baseline.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/file-size-baseline.txt b/scripts/file-size-baseline.txt index 19b6425a7..cf0dd4067 100644 --- a/scripts/file-size-baseline.txt +++ b/scripts/file-size-baseline.txt @@ -17,8 +17,8 @@ 1659 bench/terminal_bench_analysis/tests/test_tb21_evidence_contract.py 2269 crates/stella-cli/src/agent.rs 1751 crates/stella-cli/src/agent/tests.rs -4740 crates/stella-cli/src/command_deck.rs -1507 crates/stella-cli/src/fleet_cmd.rs +4691 crates/stella-cli/src/command_deck.rs +1504 crates/stella-cli/src/fleet_cmd.rs 2129 crates/stella-core/src/bus.rs 2580 crates/stella-core/src/driver.rs 3680 crates/stella-core/src/driver/tests.rs @@ -28,14 +28,14 @@ 1895 crates/stella-model/src/zai/tests.rs 3616 crates/stella-pipeline/src/pipeline.rs 2572 crates/stella-pipeline/src/pipeline/tests.rs -2963 crates/stella-protocol/src/event.rs +2965 crates/stella-protocol/src/event.rs 1995 crates/stella-store/src/lib.rs 2268 crates/stella-store/src/tests.rs 1916 crates/stella-store/src/usage.rs 1569 crates/stella-tools/src/media.rs 2184 crates/stella-tools/src/registry.rs 1839 crates/stella-tools/src/scripts.rs -1528 crates/stella-tui/src/deck_render.rs +1531 crates/stella-tui/src/deck_render.rs 4006 crates/stella-tui/src/deck_ui.rs 1665 crates/stella-tui/src/views/engine.rs 1611 crates/stella-tui/src/views/session.rs