Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions crates/stella-cli/src/agent/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,10 @@ pub(crate) fn apply_pipeline_tuning(cfg: &Config, mut config: PipelineConfig) ->
// #1291: the strict reading of an UNMEASURABLE overlap. A measured
// non-overlap withholds the deterministic credit whatever this says.
config.require_diff_coverage = engine.pipeline_require_diff_coverage_on();
// #1795: refuse a self-graded verdict where the operator said so. Absent
// is off — the single-provider seat keeps its soft path, and the verdict
// records grader independence on its snapshot either way.
config.require_independent_verifier = engine.pipeline_require_independent_verifier_on();
if let Some(candidates) = engine.pipeline_candidates {
// Stored as written, including `0`. `PipelineConfig::candidate_count`
// floors at 1, so a zero reads as single-shot rather than as "run
Expand Down
21 changes: 21 additions & 0 deletions crates/stella-cli/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,19 @@ pub struct AgentEngineConfig {
/// which is what makes the two arms one binary and one posture key apart.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_verifier_evidence_demand: Option<Toggle>,
/// Refuse a run whose VERDICT call would resolve to the worker's own
/// model (`stella_pipeline::PipelineConfig::require_independent_verifier`,
/// #1795). Absent is off.
///
/// Off is the right default for a single-provider BYOK seat, where the
/// verifier legitimately rides the worker and the verdict records that
/// fact on its ladder snapshot instead. Turn it on where the posture
/// claims an independent reviewer — a benchmark arm, a policy that treats
/// a self-graded PASS as no verdict at all — and a run that cannot honour
/// the claim should refuse before spending rather than produce a number
/// the configuration misdescribes.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_require_independent_verifier: Option<Toggle>,
/// Seconds of provider silence that end a single generation
/// (`stella_core::EngineConfig::model_timeout`). Absent keeps the engine's
/// own default, which is what every run used before this key existed.
Expand Down Expand Up @@ -729,6 +742,7 @@ impl AgentEngineConfig {
take!(pipeline_candidates);
take!(pipeline_verifier_evidence_demand);
take!(pipeline_require_diff_coverage);
take!(pipeline_require_independent_verifier);
take!(model_timeout_secs);
take!(compaction_budget_tokens);
take!(tool_result_horizon_steps);
Expand Down Expand Up @@ -836,6 +850,13 @@ impl AgentEngineConfig {
.is_some_and(Toggle::is_on)
}

/// Whether a verdict that would be graded by the worker's own model
/// refuses the run instead (#1795). Absent is off.
pub fn pipeline_require_independent_verifier_on(&self) -> bool {
self.pipeline_require_independent_verifier
.is_some_and(Toggle::is_on)
}

/// Whether interactive surfaces gate writes on per-hunk approval (#1265).
pub fn hunk_review_on(&self) -> bool {
self.hunk_review.is_some_and(Toggle::is_on)
Expand Down
4 changes: 4 additions & 0 deletions crates/stella-cli/src/settings/toml_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,8 @@ pub struct AgentsSection {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_require_diff_coverage: Option<Toggle>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_require_independent_verifier: Option<Toggle>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model_timeout_secs: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub compaction_budget_tokens: Option<u64>,
Expand Down Expand Up @@ -597,6 +599,7 @@ pub fn raise_agents(cfg: &AgentEngineConfig) -> (AgentsSection, ModelsSection) {
pipeline_candidates: cfg.pipeline_candidates,
pipeline_verifier_evidence_demand: cfg.pipeline_verifier_evidence_demand,
pipeline_require_diff_coverage: cfg.pipeline_require_diff_coverage,
pipeline_require_independent_verifier: cfg.pipeline_require_independent_verifier,
model_timeout_secs: cfg.model_timeout_secs,
compaction_budget_tokens: cfg.compaction_budget_tokens,
tool_result_horizon_steps: cfg.tool_result_horizon_steps,
Expand Down Expand Up @@ -670,6 +673,7 @@ fn lower_agents(agents: AgentsSection, models: ModelsSection) -> Option<AgentEng
pipeline_candidates: agents.pipeline_candidates,
pipeline_verifier_evidence_demand: agents.pipeline_verifier_evidence_demand,
pipeline_require_diff_coverage: agents.pipeline_require_diff_coverage,
pipeline_require_independent_verifier: agents.pipeline_require_independent_verifier,
model_timeout_secs: agents.model_timeout_secs,
compaction_budget_tokens: agents.compaction_budget_tokens,
tool_result_horizon_steps: agents.tool_result_horizon_steps,
Expand Down
1 change: 1 addition & 0 deletions crates/stella-cli/src/trace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,7 @@ mod tests {
witness_intact: None,
witness_mutation: None,
diff_coverage: None,
verifier_independent: None,
}
}
}
42 changes: 39 additions & 3 deletions crates/stella-pipeline/src/pipeline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,18 @@ pub struct PipelineConfig {
/// number at all: the arm's own digest describes a configuration the run
/// did not have (#1147).
pub require_independent_witness: bool,
/// Refuse the run when the VERDICT call would resolve to the worker's own
/// model (#1795) — the "independent code reviewer" grading the code it
/// wrote — instead of proceeding with the once-per-run prose caveat.
///
/// Off by default for the same reason its witness sibling above is: a
/// single-provider BYOK seat is the common case and must keep working.
/// On or off, the verdict's ladder snapshot records grader independence
/// as a structured fact (`LadderSnapshot::verifier_independent`), so a
/// stored verdict states it without the transcript. Checked before spend,
/// like the witness gate: a refusal after the trajectory is bought is a
/// trajectory the caller must throw away.
pub require_independent_verifier: bool,
/// Best-of-N (L-E7). `None` or `Some(1)` is single-shot (the default);
/// `Some(n)` generates n candidate executions — each in an isolated
/// snapshot of the current tree state when a
Expand Down Expand Up @@ -449,6 +461,7 @@ impl Default for PipelineConfig {
// evidence-demand-1295/README.md` for the measurement.
verifier_evidence_demand: true,
require_independent_witness: false,
require_independent_verifier: false,
candidates: None,
candidate_concurrency: None,
create_worktrees: crate::ports::WorktreePolicy::default(),
Expand Down Expand Up @@ -930,6 +943,19 @@ impl<'a> Pipeline<'a> {
total_cost,
));
}
// Same probe, second consequence (#1795): the VERDICT grader must be
// independent too when the caller says so. The probe compares the
// worker's and verifier's resolved model refs, which is exactly
// "would the verdict resolve to the worker's model".
if self.config.require_independent_verifier
&& let WitnessAuthorIndependence::Unavailable(reason) =
self.witness_author_independence()
{
return Err(PipelineRunError::new(
PipelineError::VerifierNotIndependent(reason),
total_cost,
));
}
if messages.is_empty() {
messages.push(CompletionMessage::system(DEFAULT_SYSTEM_PROMPT));
}
Expand Down Expand Up @@ -2790,7 +2816,11 @@ impl<'a> Pipeline<'a> {
state.last_verdict_diff = Some(stripped.diff.clone());
}
let mut evidence = model_verdict_evidence(&verdict);
evidence.ladder = Some(Box::new(snapshot.with_rung(verdict.rung())));
evidence.ladder = Some(Box::new(
snapshot
.with_rung(verdict.rung())
.with_verifier_independence(verdict.verifier_independent),
));
self.emit(AgentEvent::Verdict {
passed: verdict.passed,
evidence: evidence.clone(),
Expand Down Expand Up @@ -3220,12 +3250,18 @@ impl<'a> Pipeline<'a> {
Ok(verifier) if verifier.model_ref != worker.model_ref => {
WitnessAuthorIndependence::Independent
}
// Role-neutral wording on purpose (#1795): the same finding is
// framed by two different refusals (witness author, verdict
// grader) and one degradation notice, and each supplies its own
// role — a reason that named one would misname the others.
Ok(_) => WitnessAuthorIndependence::Unavailable(format!(
"no author independent of the worker (verifier and worker both resolved to `{}`)",
"no model independent of the worker resolves (verifier and worker both \
resolved to `{}`)",
worker.model_ref
)),
Err(_) => WitnessAuthorIndependence::Unavailable(
"no author independent of the worker (the verifier role is unresolvable)"
"no model independent of the worker resolves (the verifier role is \
unresolvable)"
.to_string(),
),
}
Expand Down
13 changes: 13 additions & 0 deletions crates/stella-pipeline/src/pipeline/run_error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,19 @@ pub enum PipelineError {
"an independent witness author was required but {0} — refusing rather than running as the single-model arm under a configuration that claims otherwise"
)]
WitnessAuthorUnavailable(String),
/// [`crate::PipelineConfig::require_independent_verifier`] is on and the
/// verdict call would resolve to the worker's own model — or to no model
/// at all (#1795).
///
/// Same shape and same before-spend placement as the witness refusal
/// above, for the same caller: one that has published the claim that an
/// independent reviewer grades the work. The ordinary posture keeps the
/// soft path — the verdict runs self-graded, records the fact on its
/// ladder snapshot, and the router's caveat says so in prose.
#[error(
"an independent verifier was required for the verdict but {0} — refusing before spend rather than letting the worker grade its own work under a configuration that claims otherwise"
)]
VerifierNotIndependent(String),
/// A required role (worker) could not be resolved at all.
#[error(transparent)]
Routing(#[from] RouterError),
Expand Down
8 changes: 6 additions & 2 deletions crates/stella-pipeline/src/pipeline/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1888,7 +1888,7 @@ async fn single_model_config_degrades_to_unauthored_witness_instead_of_aborting(
matches!(
event,
AgentEvent::Error { message, retryable: true }
if message.contains("no author independent of the worker")
if message.contains("no model independent of the worker")
)
});
assert!(warned, "the degradation is announced: {events:?}");
Expand All @@ -1897,7 +1897,7 @@ async fn single_model_config_degrades_to_unauthored_witness_instead_of_aborting(
event,
AgentEvent::Proof {
step: stella_protocol::ProofStep::WitnessUnavailable { reason }
} if reason.contains("no author independent of the worker")
} if reason.contains("no model independent of the worker")
)
});
assert!(
Expand Down Expand Up @@ -2382,6 +2382,10 @@ mod verification_hardening;
/// declining to ask where no tracked command could ever answer. A child
/// module, so it reaches the scripted ports above via `super::*`.
mod verifier_evidence_demand;
/// Verifier != worker for the verdict call (#1795): the opt-in refusal and
/// the structured grader-independence fact on the snapshot. A child module,
/// so it reaches the scripted ports above via `super::*`.
mod verifier_independence;
/// Proportionate verification: changes with nothing to prove complete with a
/// stated reason rather than escalating. A child module, so it reaches the
/// scripted ports above via `super::*`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ async fn unavailable_independent_witness_degrades_instead_of_aborting() {
events.iter().any(|event| matches!(
event,
AgentEvent::Error { message, retryable: true }
if message.contains("no author independent of the worker")
if message.contains("no model independent of the worker")
)),
"the degradation is announced once: {events:?}"
);
Expand Down
Loading