From 132172944c07de77c11266c953996fc1e840b647 Mon Sep 17 00:00:00 2001 From: Kitahl <222771578+Kitahl@users.noreply.github.com> Date: Wed, 26 Aug 2026 23:39:41 -0700 Subject: [PATCH] Council vNext: add challenge-derived review seats --- docs/specs/COUNCIL_ENGINEERING_SPEC.md | 193 ++++-- skills/council-of-elders/SKILL.md | 81 ++- tests/test_council_challenge_seats.py | 509 +++++++++++++++ tools/council_runtime.py | 831 ++++++++++++++++++++++--- 4 files changed, 1459 insertions(+), 155 deletions(-) create mode 100644 tests/test_council_challenge_seats.py diff --git a/docs/specs/COUNCIL_ENGINEERING_SPEC.md b/docs/specs/COUNCIL_ENGINEERING_SPEC.md index 431d1164..7276ba86 100644 --- a/docs/specs/COUNCIL_ENGINEERING_SPEC.md +++ b/docs/specs/COUNCIL_ENGINEERING_SPEC.md @@ -2,54 +2,163 @@ ## Obligation -Provide selective independent review of a concrete artifact after a strong direct pass, while measuring rather than assuming diversity/independence. +Provide selective, controlled review of a concrete artifact after a strong direct pass while measuring rather than assuming review diversity or independence. -## Current workflow +Council owns only the `REVIEW` obligation. It may produce supported findings and proposal-only neutral challenges for target modules, but it cannot clear a `PROOF`, `DISCOVERY`, `SYNTHESIS`, `ENGINEERING`, `EVALUATION`, `ASSURANCE`, `PREFLIGHT`, `ADAPTATION`, or `ADVERSARY` obligation. -Specification: 3–6 artifact-derived seats, skeptic, frozen independent first passes, disjoint evidence where feasible, reveal/cross-critique, evidence-ranked synthesis, matched direct control. +## Runtime and schemas -Closest executable helper currently uses generic paired red-team prompts and synthesis; it does not enforce the full Council contract. +- Runtime: `tools/council_runtime.py` +- Domain receipt schema: existing `egrt.runtime.v1` +- Challenge schema: additive `egrt.challenge.v1` +- Neutral challenge implementation: `tools/egrt_challenge_types.py` and `tools/egrt_challenge.py` +- State machine: `COMMIT -> REVEAL -> CROSS_CRITIQUE -> CLOSED` -## vNext state machine +Council never imports `foil_*`. FOIL remains an optional later controller and is not part of the native Council path. -`COMMIT -> REVEAL -> CROSS_CRITIQUE -> CLOSED` +## Frozen Council state A Council has: -- artifact hash and frozen total-budget hash; -- 3–6 distinct seat questions; -- role/method/evidence partition per seat; +- an optional bound task ID; +- a frozen artifact hash and frozen total-budget hash; +- 3–6 seats; - first-pass commitment hashes; -- sealed submissions; -- revealed submissions; -- provenance/evidence/method/finding overlap diagnostics; -- structured cross-critique records, with every seat contributing after reveal; -- DIRECT and optional VOTE control receipt IDs. - -## vNext workflow - -1. Confirm concrete artifact and existing direct analysis. -2. Derive 3–6 seats from distinct failure/verification obligations. -3. Freeze seat role/question/method/evidence partition. -4. Each seat commits first-pass submission hash before reveal. -5. Reveal only when all commits are frozen; tampered reveals fail hash verification. -6. Compute evidence/provenance/method/finding overlap. -7. Cross-critique only after reveal. -8. Require each seat to contribute a structured cross-critique against another revealed seat. -9. Synthesize by claims/evidence, preserving disagreement. -10. Compare against a DIRECT control on the same frozen artifact and matching total-budget hash; optionally compare VOTE. -11. Council's *marginal-value* claim remains UNKNOWN without same-artifact/same-budget direct-control evidence. -12. Record confidence as uncalibrated unless prospective Brier/log-score calibration exists. - -## Runtime - -`tools/council_runtime.py` - -## Mechanical tests - -- reject <3 or >6 seats; -- reject duplicate seat questions; -- double commit forbidden; -- tampered reveal rejected; -- overlap matrix correct; -- missing direct control keeps marginal-value verdict UNKNOWN. +- sealed and revealed submissions; +- structured cross-critiques; +- method, evidence, provenance, reviewer-provenance, and finding-overlap diagnostics; +- structured Council findings; +- supported finding IDs selected at synthesis; +- a mandatory same-artifact, same-total-budget `DIRECT` control receipt; +- an optional `VOTE` control receipt; +- one Council receipt that can clear only the bound `REVIEW` obligation. + +## Challenge-derived seat contract + +A vNext `CouncilSeat` extends the legacy seat fields with: + +- `challenge_kind`; +- `discriminator`; +- `required_capability`; +- `target_obligation_id`; +- `refuter`; +- `reviewer_provenance`; +- `challenge_contract="vnext"`. + +Legacy four-field seats remain readable and executable for backward compatibility. A Council cannot mix legacy and vNext seats in one run. + +### Residual-gap seat classes + +Derive seats from unresolved neutral challenge classes rather than generic personalities: + +| Residual class | Typical role | Decisive partition or discriminator | +|---|---|---| +| Formal correctness | proof/formalization | exact derivation, contradiction, alternate formalization, counterexample | +| Evidence and provenance | source assessment | first-party evidence, source conflict, citation/provenance lineage | +| Executable behavior | implementation/integration | real entrypoint, failure class, output contract, metamorphic relation | +| Measurement validity | evaluation design | baseline/estimand, contamination, scorer, stopping or budget boundary | +| Novelty or costume | prior-art/ablation | strongest known costume, assumption knockout, transfer or ablation | +| Operational feasibility | state/decision/operations | state drift, resource boundary, reversal condition | + +`derive_challenge_seats()` deterministically converts open `ChallengeRequest` objects into vNext seats. Resolved and dismissed challenges are ignored. Load-bearing, higher-risk, higher-information, and lower-cost challenges are prioritized within the frozen maximum of six seats. + +Every seat must own a distinct question. Duplicate challenge kinds or duplicate discriminators are rejected unless both seats declare genuinely distinct evidence partitions. A skeptic/adversarial seat is mandatory and must name a concrete refuter; a generic instruction such as “be critical” does not satisfy the contract. + +## Commit–reveal and cross-critique + +1. Freeze the artifact, total budget, seats, questions, methods, evidence partitions, challenge bindings, and reviewer provenance. +2. Each seat commits a salted hash of its structured first-pass submission before any reveal. +3. Reveal only after all commitments are frozen. A wrong nonce or altered submission fails verification. +4. Compute evidence, provenance, method, reviewer-provenance, and finding overlap. +5. Cross-critique only after every reveal. +6. Require every seat to contribute a structured critique against another seat. +7. Synthesize by supported finding IDs and preserve unresolved disagreement. + +Nominal seat count never establishes statistical independence. Every pair reports `independence_status="NOT_ESTABLISHED"`, and the Council-level diagnostic is `NOT_ESTABLISHED_BY_SEAT_COUNT`. Same-model or same-reviewer provenance remains explicit. + +## Structured findings and neutral challenge output + +A supported vNext finding is recorded as `CouncilFinding` with: + +- seat ID; +- target module and target obligation ID; +- challenge kind; +- hypothesis, alternative, concrete refuter, and consequence if true; +- load-bearing flag and required capability; +- evidence partition; +- candidate, scope, and obligation-set SHA-256 bindings; +- provenance and typed metadata. + +`record_finding()` is permitted only after all seats reveal. For vNext seats, the finding must match the seat’s challenge kind, target obligation, required capability, concrete refuter, and evidence partition, and its finding ID must have appeared in that seat’s frozen revealed submission. + +A vNext Council cannot clear its `REVIEW` obligation with a supported finding that lacks a structured `CouncilFinding` record. + +After Council finalization, `propose_supported_finding_challenge()` may convert one supported structured finding into a neutral `ChallengeRequest` with `origin=COUNCIL`. The emitted object is: + +- bound to the same task, target obligation, candidate, scope, and obligation set; +- stored initially as `PROPOSED`; +- marked `authority=PROPOSAL_ONLY`; +- unable to resolve itself; +- unable to create a domain receipt; +- unable to clear the target module’s obligation; +- subject to the shared challenge budget, deduplication, selection, and release rules. + +The target module or host must select a discriminator and the claim-native module must produce the resolving evidence. + +## Control and release semantics + +A Council review requires a real `DIRECT` control receipt with the same: + +- review obligation ID; +- frozen artifact hash; +- frozen total-budget hash. + +The optional `VOTE` control is diagnostic only. + +Council finalization returns: + +- `UNKNOWN` when commit/reveal is incomplete; +- `UNKNOWN` when cross-critique participation is incomplete; +- `UNKNOWN` when the matched `DIRECT` control is absent, invalid, or mismatched; +- `UNKNOWN` when a vNext supported finding lacks a structured record; +- `UNKNOWN` when synthesis supports no findings; +- `CLEARED` only when the frozen Council review protocol completed and at least one supported finding remains. + +`CLEARED` means only that the controlled Council protocol completed for the `REVIEW` obligation. It is not a verdict on the target domain claim and not evidence that Council outperforms `DIRECT` or `VOTE`. + +When a bound task exists, attempting to finalize Council against a non-`REVIEW` obligation raises `CouncilAuthorityError`. A Council receipt records `authority=REVIEW_ONLY` and `target_domain_clearance_authorized=false`. + +## Backward compatibility + +- Existing `CouncilSeat(seat_id, role, question, method, evidence_partition=None)` construction remains valid. +- Existing commit/reveal, overlap, cross-critique, control, and finalization behavior remains valid for legacy Council state. +- Historical Council state loads through inferred legacy challenge fields; historical receipts are not rewritten. +- Native Council challenge behavior is additive and uses the shared challenge configuration. +- No FOIL profile, FOIL receipt, or seat count changes evidence authority. + +## Mechanical acceptance tests + +`tests/test_council_challenge_seats.py` proves: + +- distinct challenge questions and duplicate-kind/discriminator rejection; +- deterministic derivation from open challenge classes; +- concrete skeptic/refuter behavior; +- structured supported finding to neutral `COUNCIL` challenge conversion; +- target-domain authority denial; +- mandatory same-artifact/same-budget `DIRECT` control; +- same-model seats are never labeled independent solely by seat count. + +Existing Council tests continue to prove: + +- 3–6 seat bounds; +- skeptic requirement; +- commitment hiding and binding; +- tampered reveal rejection; +- overlap diagnostics; +- complete cross-critique participation; +- matching control semantics; +- non-recallable finalization. + +## Non-goals and efficacy boundary + +This work does not establish that Council improves task outcomes. It does not treat agreement as truth, seat count as independence, or a Council finding as claim-native evidence. Marginal-value claims still require prospective, matched-budget comparison against `DIRECT`; null and negative results must be retained. diff --git a/skills/council-of-elders/SKILL.md b/skills/council-of-elders/SKILL.md index 74ac2144..30df96b7 100644 --- a/skills/council-of-elders/SKILL.md +++ b/skills/council-of-elders/SKILL.md @@ -1,11 +1,11 @@ --- name: council-of-elders -description: Evidence Review Panel. Trigger: /council, /forum, "independent review", or equivalent. Selectively convenes independent, evidence-grounded review roles around a concrete artifact after a strong direct pass exists. Uses commit-reveal, disjoint evidence where possible, a skeptic, and a matched direct control. Agreement is not truth. +description: Evidence Review Panel. Trigger: /council, /forum, "independent review", or equivalent. Selectively convenes challenge-derived, evidence-grounded review seats around a concrete artifact after a strong direct pass exists. Uses commit-reveal, distinct discriminators or evidence partitions, a concrete skeptic refuter, cross-critique, overlap diagnostics, and a matched direct control. Agreement is not truth. --- # Evidence Review Panel -The public panel uses **artifact-derived review roles**, not simulated authority from named personalities. +Use artifact-derived review seats, not named personalities or simulated authority. ## When to convene @@ -13,41 +13,72 @@ Default: **off**. Convene only when: -- there is a concrete artifact/claim/design to review; +- there is a concrete artifact, claim, design, or candidate to review; - a strong direct analysis already exists; -- independent evidence/methods are plausibly available; -- the marginal value can be compared against a direct control. +- at least three genuinely different challenge questions, discriminators, or evidence partitions are available; +- the total artifact and review budget can be frozen; +- marginal value can be compared against a same-artifact, same-budget direct control. -Do not convene merely because the problem is hard or more opinions sound reassuring. +Do not convene merely because a problem is difficult or additional opinions sound reassuring. -## Seat construction +## Construct challenge-derived seats -Prefer 3–6 roles derived from the artifact, for example: +Prefer 3–6 seats derived from unresolved load-bearing gaps: - formal correctness; -- empirical/measurement validity; -- implementation/integration; -- adversarial failure modes; -- prior art/novelty; -- cost/operational feasibility. +- evidence and provenance; +- executable behavior or integration; +- measurement validity; +- novelty or strongest costume; +- operational feasibility. -Include a skeptic role. Add a role only when it has a distinct question, evidence pack, verifier, or method. +Every seat must own: -## Commit-reveal +- one distinct challenge question; +- one concrete discriminator or refuter; +- one required capability when applicable; +- one target obligation; +- one method or explicit evidence partition. -1. Each role independently freezes its first-pass conclusion before seeing other roles. -2. Use disjoint searches/evidence where feasible. -3. Reveal and cross-critique after independent commits. -4. Normalize claims to the same scope. -5. Rank by evidence, not identity or vote count. -6. Preserve unresolved disagreement. +Duplicate challenge kinds or discriminators are allowed only when the evidence partitions are explicitly different. Include a skeptic/adversarial seat with a concrete refuter; “be critical” is not a discriminator. -## Control +## Commit–reveal and cross-critique -Run one strong direct analysis with access to the same combined evidence budget when feasible. Attribute panel value only to findings the control did not produce or to stronger verification/coverage. +1. Freeze the artifact hash, total-budget hash, seats, methods, challenge bindings, evidence partitions, and reviewer provenance. +2. Each seat independently commits its first-pass structured submission before seeing another seat’s result. +3. Reveal only after all commitments are frozen. +4. Reject altered submissions or wrong nonces. +5. Compute method, evidence, provenance, reviewer-provenance, and finding overlap. +6. Require every seat to cross-critique another revealed seat. +7. Synthesize by claim and evidence, preserving disagreement. -Panel output remains a claim until independently checked when the conclusion is load-bearing. +Do not call seats independent because there are several of them or because they produced separate messages. Same-model and shared-evidence common causes remain explicit. + +## Supported findings + +A load-bearing supported finding remains a review finding until the target module checks it. + +The Council runtime may convert a supported structured finding into an additive neutral challenge with `origin=COUNCIL`. That challenge is proposal-only. It cannot resolve itself, replace a domain receipt, clear the target obligation, apply a repair, or authorize a write. + +## Control and authority + +Run a strong direct analysis against the same frozen artifact and matching total budget. Without that `DIRECT` control, Council’s review verdict remains `UNKNOWN`. + +A Council receipt may clear only a `REVIEW` obligation. It never clears proof, discovery, synthesis, engineering, evaluation, assurance, preflight, adaptation, or adversary obligations. User or host authority remains required for adoption or writes. ## Typed runtime contract -`tools/council_runtime.py` enforces 3–6 distinct seat questions, a skeptic/adversarial seat, frozen commitment hashes before reveal, reveal integrity, and evidence/provenance overlap diagnostics. Council's REVIEW verdict remains `UNKNOWN` until commit/reveal is complete, every seat contributes cross-critique, and a real DIRECT control matches the same frozen artifact and total-budget hashes. Confidence is recorded as uncalibrated unless later scored prospectively. See `docs/specs/COUNCIL_ENGINEERING_SPEC.md`. +`tools/council_runtime.py` enforces: + +- 3–6 seats and a skeptic/adversarial seat; +- challenge kind, discriminator, capability, target obligation, refuter, and evidence partition for vNext seats; +- rejection of duplicate challenge/discriminator seats without distinct partitions; +- commitment hiding and reveal integrity; +- complete cross-critique participation; +- overlap diagnostics with `NOT_ESTABLISHED_BY_SEAT_COUNT`; +- structured `CouncilFinding` binding; +- proposal-only `COUNCIL` challenge emission; +- `REVIEW_ONLY` receipt authority; +- a same-artifact, same-total-budget `DIRECT` control. + +Confidence remains uncalibrated unless scored prospectively. See `docs/specs/COUNCIL_ENGINEERING_SPEC.md`. diff --git a/tests/test_council_challenge_seats.py b/tests/test_council_challenge_seats.py new file mode 100644 index 00000000..3829fe6b --- /dev/null +++ b/tests/test_council_challenge_seats.py @@ -0,0 +1,509 @@ +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) + +import council_runtime as council # noqa: E402 +from egrt_challenge_types import ( # noqa: E402 + ChallengeKind, + ChallengeOrigin, + ChallengeRequest, +) +from egrt_store import RuntimeStore # noqa: E402 +from egrt_types import ( # noqa: E402 + Obligation, + ObligationKind, + TaskState, + Verdict, + digest, +) + + +def init_root(path: Path) -> None: + (path / ".gauntlet.json").write_text( + json.dumps({"state_dir": ".egrt/state", "runtime": {"enabled": True}}), + encoding="utf-8", + ) + + +class CouncilChallengeSeatTests(unittest.TestCase): + def _task(self, root: Path) -> tuple[str, str, str]: + task_id = "task-council-vnext" + review_id = "obl-review" + target_id = "obl-proof" + RuntimeStore(root).write_task( + TaskState( + task_id=task_id, + goal_hash=digest("review a proof candidate"), + obligations=[ + Obligation( + review_id, + ObligationKind.REVIEW, + "Run the controlled review protocol", + required_module="council", + ), + Obligation( + target_id, + ObligationKind.PROOF, + "Prove the bound candidate", + required_module="mind", + ), + ], + ) + ) + return task_id, review_id, target_id + + def _seats( + self, + target_id: str, + *, + shared_reviewer: bool = False, + ) -> list[council.CouncilSeat]: + provenance = "same-model-v1" if shared_reviewer else None + return [ + council.CouncilSeat( + "s1", + "formal correctness", + "Do the competing quantifier scopes agree?", + "exact proof", + "formal-evidence", + ChallengeKind.ALTERNATE_FORMALIZATION, + "derive both formalizations exactly", + "FORMAL_PROOF", + target_id, + "exhibit the scope whose exact result differs", + provenance, + ), + council.CouncilSeat( + "s2", + "evidence / provenance", + "Does the source record support the encoded premise?", + "source assessment", + "source-evidence", + ChallengeKind.SOURCE_CONFLICT, + "assess the first-party provenance chain", + "SCHOLARLY_SEARCH", + target_id, + "identify the conflicting first-party record", + provenance, + ), + council.CouncilSeat( + "s3", + "skeptic", + "Does a finite counterexample refute the candidate?", + "bounded adversarial search", + "counterexample-evidence", + ChallengeKind.COUNTEREXAMPLE, + "enumerate the smallest witness domain", + "FORMAL_PROOF", + target_id, + "produce the concrete witness and failed predicate", + "different-model-v2" if shared_reviewer else None, + ), + ] + + def _run_review( + self, + root: Path, + target_id: str, + *, + task_id: str | None = None, + shared_reviewer: bool = False, + ) -> council.CouncilState: + seats = self._seats(target_id, shared_reviewer=shared_reviewer) + state = council.create_council( + root, + "artifact-hash", + "budget-hash", + seats, + task_id=task_id, + ) + submissions = { + "s1": council.SeatSubmission( + "scope may differ", + ("c1",), + ("e1",), + ("p1",), + 0.7, + ("finding-scope",), + ), + "s2": council.SeatSubmission( + "source chain may conflict", + ("c2",), + ("e2",), + ("p2",), + 0.6, + ("finding-source",), + ), + "s3": council.SeatSubmission( + "counterexample may exist", + ("c3",), + ("e3",), + ("p3",), + 0.8, + ("finding-counterexample",), + ), + } + nonces: dict[str, str] = {} + for seat_id, submission in submissions.items(): + _, nonces[seat_id] = council.commit( + root, + state.council_id, + seat_id, + submission, + ) + for seat_id, submission in submissions.items(): + self.assertTrue( + council.reveal( + root, + state.council_id, + seat_id, + submission, + nonces[seat_id], + ) + ) + council.record_cross_critique( + root, + state.council_id, + council.CrossCritique( + "s1", + "s2", + surviving_findings=("finding-source",), + ), + ) + council.record_cross_critique( + root, + state.council_id, + council.CrossCritique( + "s2", + "s3", + surviving_findings=("finding-counterexample",), + ), + ) + council.record_cross_critique( + root, + state.council_id, + council.CrossCritique( + "s3", + "s1", + surviving_findings=("finding-scope",), + ), + ) + return state + + def test_distinct_seat_challenges(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + duplicate = [ + council.CouncilSeat( + "a", + "skeptic", + "q1", + "m1", + None, + ChallengeKind.COUNTEREXAMPLE, + "same discriminator", + "FORMAL_PROOF", + "target", + "specific refuter one", + ), + council.CouncilSeat( + "b", + "formal correctness", + "q2", + "m2", + None, + ChallengeKind.COUNTEREXAMPLE, + "different discriminator", + "FORMAL_PROOF", + "target", + "specific refuter two", + ), + council.CouncilSeat( + "c", + "measurement validity", + "q3", + "m3", + "partition-c", + ChallengeKind.BASELINE_OR_ESTIMAND, + "design check", + "STATISTICAL_ANALYSIS", + "target", + "compare the frozen direct baseline", + ), + ] + with self.assertRaises(ValueError): + council.create_council(root, "artifact", "budget", duplicate) + + state = council.create_council( + root, + "artifact", + "budget", + self._seats("target"), + ) + self.assertEqual( + {seat.challenge_contract for seat in state.seats}, + {"vnext"}, + ) + self.assertEqual( + len({seat.question for seat in state.seats}), + len(state.seats), + ) + + def test_generic_skeptic_refuter_rejected(self) -> None: + with self.assertRaises(ValueError): + council.CouncilSeat( + "s3", + "skeptic", + "Does the candidate survive a concrete attack?", + "bounded adversarial search", + "counterexample-evidence", + ChallengeKind.COUNTEREXAMPLE, + "enumerate the smallest witness domain", + "FORMAL_PROOF", + "target", + "be critical", + ) + + def test_council_finding_creates_challenge(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + task_id, review_id, target_id = self._task(root) + state = self._run_review(root, target_id, task_id=task_id) + finding = council.CouncilFinding( + finding_id="finding-scope", + seat_id="s1", + target_module="mind", + target_obligation_id=target_id, + challenge_kind=ChallengeKind.ALTERNATE_FORMALIZATION, + hypothesis="The natural-language quantifier scope may encode a different theorem", + refuter="exhibit the scope whose exact result differs", + consequence_if_true="The current proof candidate addresses the wrong scoped claim", + candidate_hash=digest("candidate"), + scope_hash=digest("scope"), + obligation_set_hash=digest("obligation-set"), + required_capability="FORMAL_PROOF", + evidence_partition="formal-evidence", + ) + council.record_finding(root, state.council_id, finding) + direct = council.record_control( + root, + review_id, + artifact_hash="artifact-hash", + budget_hash="budget-hash", + kind="DIRECT", + output_hash=digest("direct"), + verdict=Verdict.CLEARED, + verifier="direct-review", + ) + receipt = council.finalize( + root, + state.council_id, + review_id, + synthesis_hash=digest("synthesis"), + supported_findings=["finding-scope"], + direct_control_receipt=direct.receipt_id, + ) + self.assertEqual(receipt.verdict, Verdict.CLEARED) + + request = council.propose_supported_finding_challenge( + root, + state.council_id, + "finding-scope", + ) + self.assertEqual(request.origin, ChallengeOrigin.COUNCIL) + self.assertEqual(request.obligation_id, target_id) + self.assertEqual(request.target_module, "mind") + self.assertFalse(request.metadata["domain_clearance_authorized"]) + stored = RuntimeStore(root).read_challenge(request.challenge_id) + self.assertIsNotNone(stored) + self.assertEqual(stored["state"], "PROPOSED") + self.assertFalse( + any( + row.get("module") == "council" + for row in RuntimeStore(root).receipts_for(target_id) + ) + ) + + def test_domain_authority_denied(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + task_id, _, target_id = self._task(root) + state = self._run_review(root, target_id, task_id=task_id) + direct = council.record_control( + root, + target_id, + artifact_hash="artifact-hash", + budget_hash="budget-hash", + kind="DIRECT", + output_hash=digest("direct"), + verdict=Verdict.CLEARED, + verifier="direct-review", + ) + with self.assertRaises(council.CouncilAuthorityError): + council.finalize( + root, + state.council_id, + target_id, + synthesis_hash=digest("synthesis"), + supported_findings=["finding-scope"], + direct_control_receipt=direct.receipt_id, + ) + + def test_vnext_supported_finding_requires_structured_record(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + task_id, review_id, target_id = self._task(root) + state = self._run_review(root, target_id, task_id=task_id) + direct = council.record_control( + root, + review_id, + artifact_hash="artifact-hash", + budget_hash="budget-hash", + kind="DIRECT", + output_hash=digest("direct"), + verdict=Verdict.CLEARED, + verifier="direct-review", + ) + receipt = council.finalize( + root, + state.council_id, + review_id, + synthesis_hash=digest("synthesis"), + supported_findings=["finding-scope"], + direct_control_receipt=direct.receipt_id, + ) + self.assertEqual(receipt.verdict, Verdict.UNKNOWN) + self.assertTrue( + any( + item.startswith("supported-finding-missing-structured-record:") + for item in receipt.unresolved + ) + ) + + def test_direct_control_still_required(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + task_id, review_id, target_id = self._task(root) + state = self._run_review(root, target_id, task_id=task_id) + receipt = council.finalize( + root, + state.council_id, + review_id, + synthesis_hash=digest("synthesis"), + supported_findings=["finding-scope"], + ) + self.assertEqual(receipt.verdict, Verdict.UNKNOWN) + self.assertTrue( + any("direct-control" in item for item in receipt.unresolved) + ) + + def test_same_model_seats_are_not_independent_by_count(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + init_root(root) + state = self._run_review( + root, + "target", + shared_reviewer=True, + ) + matrix = council.overlap_matrix( + council._load(root, state.council_id) + ) + self.assertEqual( + matrix["diagnostics"]["independence_status"], + "NOT_ESTABLISHED_BY_SEAT_COUNT", + ) + self.assertGreaterEqual( + matrix["diagnostics"]["same_reviewer_provenance_pairs"], + 1, + ) + self.assertTrue( + all( + row["independence_status"] == "NOT_ESTABLISHED" + for row in matrix["pairs"] + ) + ) + + def test_derive_seats_from_open_challenges(self) -> None: + hashes = { + "candidate_hash": digest("candidate"), + "scope_hash": digest("scope"), + "obligation_set_hash": digest("set"), + } + rows = [ + ChallengeRequest( + challenge_id="c-formal", + task_id="task", + obligation_id="target", + target_module="mind", + origin=ChallengeOrigin.MODULE_NATIVE, + kind=ChallengeKind.ALTERNATE_FORMALIZATION, + hypothesis="Two formalizations may disagree", + alternative="alternate scope", + refuter="derive both exactly", + consequence_if_true="candidate scope is wrong", + load_bearing=True, + required_capability="FORMAL_PROOF", + proposer="mind", + **hashes, + ), + ChallengeRequest( + challenge_id="c-source", + task_id="task", + obligation_id="target", + target_module="space", + origin=ChallengeOrigin.MODULE_NATIVE, + kind=ChallengeKind.SOURCE_CONFLICT, + hypothesis="Sources may share derivative provenance", + alternative=None, + refuter="inspect first-party lineage", + consequence_if_true="support is not independent", + load_bearing=True, + required_capability="SCHOLARLY_SEARCH", + proposer="space", + **hashes, + ), + ChallengeRequest( + challenge_id="c-exec", + task_id="task", + obligation_id="target", + target_module="power", + origin=ChallengeOrigin.MODULE_NATIVE, + kind=ChallengeKind.FAILURE_CLASS, + hypothesis="The real entrypoint may fail", + alternative=None, + refuter="execute the real entrypoint", + consequence_if_true="candidate is not operational", + load_bearing=True, + required_capability="CODE_EXECUTION", + proposer="power", + **hashes, + ), + ] + seats = council.derive_challenge_seats(rows) + self.assertEqual(len(seats), 3) + self.assertTrue( + any("skeptic" in seat.role.lower() for seat in seats) + ) + self.assertTrue( + all(seat.target_obligation_id == "target" for seat in seats) + ) + self.assertTrue( + all(seat.challenge_contract == "vnext" for seat in seats) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/council_runtime.py b/tools/council_runtime.py index bb2e60c9..13304edd 100644 --- a/tools/council_runtime.py +++ b/tools/council_runtime.py @@ -1,19 +1,194 @@ -"""Evidence Review Panel state machine with commit/reveal and control matching. +"""Evidence Review Panel state machine with challenge-derived seats. Council measures rather than assumes review diversity. A cleared Council receipt requires complete commit/reveal, cross-critique participation, and a same-artifact, -same-budget DIRECT control receipt. Agreement alone never clears the obligation. +same-budget DIRECT control receipt. Council findings may propose neutral challenges, +but they never clear the target module's domain obligation. """ from __future__ import annotations import json +import re import secrets from dataclasses import dataclass, field from pathlib import Path -from typing import Any +from typing import Any, Mapping, Sequence +from egrt_challenge import ChallengeError, propose_challenge +from egrt_challenge_types import ChallengeKind, ChallengeOrigin, ChallengeRequest from egrt_store import RuntimeStore, new_id, utcnow -from egrt_types import EvidenceClass, EvidenceRef, Receipt, Verdict, digest +from egrt_types import EvidenceClass, EvidenceRef, ObligationKind, Receipt, Verdict, digest + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_TERMINAL_CHALLENGE_STATES = {"RESOLVED", "DISMISSED_NOT_APPLICABLE"} +_GENERIC_REFUTERS = {"be critical", "review critically", "what breaks it", "look for problems"} + +_RESIDUAL_SEAT_CLASSES: dict[ChallengeKind, tuple[str, str, str, str | None]] = { + ChallengeKind.ALTERNATE_FORMALIZATION: ( + "formal correctness", + "proof", + "compare formalizations with a claim-native derivation or counterexample", + "FORMAL_PROOF", + ), + ChallengeKind.CLAIM_NEGATION: ( + "formal correctness", + "proof", + "attempt the bound negation or contradiction check", + "FORMAL_PROOF", + ), + ChallengeKind.COUNTEREXAMPLE: ( + "skeptic", + "adversarial", + "produce or rule out a concrete counterexample", + "FORMAL_PROOF", + ), + ChallengeKind.ASSUMPTION_KNOCKOUT: ( + "novelty / costume", + "ablation", + "remove the named assumption and compare the bound result", + None, + ), + ChallengeKind.REPRESENTATION_SWAP: ( + "formal correctness", + "alternate representation", + "re-express the claim and compare the exact scoped result", + "REASONING", + ), + ChallengeKind.SOURCE_CONFLICT: ( + "evidence / provenance", + "source assessment", + "resolve the conflict using source quality and provenance", + "SCHOLARLY_SEARCH", + ), + ChallengeKind.RETRIEVAL_REFRAME: ( + "evidence / provenance", + "retrieval reframe", + "run the non-redundant mechanism-level query reframe", + "WEB_SEARCH", + ), + ChallengeKind.NOVELTY_COSTUME: ( + "novelty / costume", + "prior-art comparison", + "compare against the strongest known costume with assessed sources", + "SCHOLARLY_SEARCH", + ), + ChallengeKind.FAILURE_CLASS: ( + "executable behavior", + "targeted execution", + "run the smallest check that exposes the named failure class", + "CODE_EXECUTION", + ), + ChallengeKind.METAMORPHIC_RELATION: ( + "executable behavior", + "metamorphic check", + "execute the declared relation and compare outputs", + "CODE_EXECUTION", + ), + ChallengeKind.BASELINE_OR_ESTIMAND: ( + "measurement validity", + "design comparison", + "compare the frozen baseline or estimand before interpreting outcomes", + "STATISTICAL_ANALYSIS", + ), + ChallengeKind.CONTAMINATION: ( + "measurement validity", + "contamination audit", + "audit the frozen item, context, and exclusion boundary", + "STATISTICAL_ANALYSIS", + ), + ChallengeKind.STATE_DRIFT: ( + "operational feasibility", + "state audit", + "replay the bound state and detect drift", + "REPOSITORY", + ), + ChallengeKind.DECISION_REVERSAL: ( + "operational feasibility", + "reversal check", + "run the smallest information action that could reverse the decision", + None, + ), + ChallengeKind.REVIEW_DIVERSITY: ( + "review diversity", + "overlap audit", + "compare method, evidence, provenance, and finding overlap", + None, + ), + ChallengeKind.OUTPUT_CONTRACT: ( + "executable behavior", + "contract check", + "execute the output contract against the real entrypoint", + "CODE_EXECUTION", + ), +} + + +def _require_text(name: str, value: object) -> None: + if not isinstance(value, str): + raise TypeError(f"{name} must be str") + if not value.strip(): + raise ValueError(f"{name} must be non-empty") + + +def _optional_text(name: str, value: object) -> None: + if value is not None: + _require_text(name, value) + + +def _kind(value: ChallengeKind | str | None) -> ChallengeKind | None: + if value is None: + return None + if isinstance(value, ChallengeKind): + return value + if not isinstance(value, str): + raise TypeError("challenge_kind must be ChallengeKind, str, or None") + try: + return ChallengeKind(value) + except ValueError as error: + raise ValueError(f"unknown challenge_kind: {value}") from error + + +def _infer_kind(role: str) -> ChallengeKind: + value = role.lower() + if "skeptic" in value or "adversarial" in value: + return ChallengeKind.COUNTEREXAMPLE + if "formal" in value or "logic" in value or "proof" in value: + return ChallengeKind.ALTERNATE_FORMALIZATION + if "evidence" in value or "provenance" in value or "source" in value: + return ChallengeKind.SOURCE_CONFLICT + if "empirical" in value or "measurement" in value or "evaluation" in value: + return ChallengeKind.BASELINE_OR_ESTIMAND + if "implementation" in value or "integration" in value or "executable" in value: + return ChallengeKind.FAILURE_CLASS + if "novel" in value or "prior art" in value or "costume" in value: + return ChallengeKind.NOVELTY_COSTUME + if "operat" in value or "feasib" in value or "cost" in value or "ops" in value: + return ChallengeKind.STATE_DRIFT + return ChallengeKind.REVIEW_DIVERSITY + + +def _norm(value: str | None) -> str: + return " ".join((value or "").strip().lower().split()) + + +@dataclass(frozen=True) +class SeatChallengeSpec: + challenge_kind: ChallengeKind | str + discriminator: str + required_capability: str | None + target_obligation_id: str + refuter: str + evidence_partition: str | None = None + + def __post_init__(self) -> None: + challenge_kind = _kind(self.challenge_kind) + if challenge_kind is None: + raise ValueError("challenge_kind is required") + object.__setattr__(self, "challenge_kind", challenge_kind) + for name in ("discriminator", "target_obligation_id", "refuter"): + _require_text(name, getattr(self, name)) + for name in ("required_capability", "evidence_partition"): + _optional_text(name, getattr(self, name)) @dataclass(frozen=True) @@ -23,6 +198,61 @@ class CouncilSeat: question: str method: str evidence_partition: str | None = None + challenge_kind: ChallengeKind | str | None = None + discriminator: str | None = None + required_capability: str | None = None + target_obligation_id: str | None = None + refuter: str | None = None + reviewer_provenance: str | None = None + challenge_contract: str = "auto" + + def __post_init__(self) -> None: + for name in ("seat_id", "role", "question", "method"): + _require_text(name, getattr(self, name)) + for name in ( + "evidence_partition", + "discriminator", + "required_capability", + "target_obligation_id", + "refuter", + "reviewer_provenance", + ): + _optional_text(name, getattr(self, name)) + if self.challenge_contract not in {"auto", "legacy", "vnext"}: + raise ValueError("challenge_contract must be auto, legacy, or vnext") + + explicit = any( + value is not None + for value in ( + self.challenge_kind, + self.discriminator, + self.required_capability, + self.target_obligation_id, + self.refuter, + ) + ) + contract = "vnext" if self.challenge_contract == "auto" and explicit else self.challenge_contract + if contract == "auto": + contract = "legacy" + object.__setattr__(self, "challenge_contract", contract) + + challenge_kind = _kind(self.challenge_kind) or _infer_kind(self.role) + object.__setattr__(self, "challenge_kind", challenge_kind) + object.__setattr__(self, "discriminator", self.discriminator or self.method) + object.__setattr__(self, "refuter", self.refuter or self.discriminator or self.question) + + if contract == "vnext": + if not self.target_obligation_id: + raise ValueError("vNext Council seat requires target_obligation_id") + if not self.discriminator: + raise ValueError("vNext Council seat requires a discriminator") + if not self.refuter: + raise ValueError("vNext Council seat requires a concrete refuter") + if ( + ("skeptic" in self.role.lower() or "adversarial" in self.role.lower()) + and _norm(self.refuter) in _GENERIC_REFUTERS + ): + raise ValueError("vNext skeptic seat requires a concrete refuter") @dataclass(frozen=True) @@ -54,6 +284,53 @@ def __post_init__(self) -> None: raise ValueError("cross-critique must contain structured finding IDs or a critique hash") +@dataclass(frozen=True) +class CouncilFinding: + finding_id: str + seat_id: str + target_module: str + target_obligation_id: str + challenge_kind: ChallengeKind | str + hypothesis: str + refuter: str + consequence_if_true: str + candidate_hash: str + scope_hash: str + obligation_set_hash: str + alternative: str | None = None + load_bearing: bool = True + required_capability: str | None = None + evidence_partition: str | None = None + proposer_provenance: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + for name in ( + "finding_id", + "seat_id", + "target_module", + "target_obligation_id", + "hypothesis", + "refuter", + "consequence_if_true", + ): + _require_text(name, getattr(self, name)) + for name in ("alternative", "required_capability", "evidence_partition", "proposer_provenance"): + _optional_text(name, getattr(self, name)) + challenge_kind = _kind(self.challenge_kind) + if challenge_kind is None: + raise ValueError("challenge_kind is required") + object.__setattr__(self, "challenge_kind", challenge_kind) + if not isinstance(self.load_bearing, bool): + raise TypeError("load_bearing must be bool") + if not isinstance(self.metadata, dict) or any(not isinstance(key, str) for key in self.metadata): + raise TypeError("metadata must be a dict with string keys") + for name in ("candidate_hash", "scope_hash", "obligation_set_hash"): + value = getattr(self, name) + if not isinstance(value, str) or not _SHA256.fullmatch(value): + raise ValueError(f"{name} must be a lowercase 64-character SHA-256 digest") + + @dataclass class CouncilState: council_id: str @@ -68,45 +345,165 @@ class CouncilState: cross_critiques: list[dict[str, Any]] = field(default_factory=list) direct_control_receipt: str | None = None vote_control_receipt: str | None = None + findings: dict[str, dict[str, Any]] = field(default_factory=dict) + supported_finding_ids: list[str] = field(default_factory=list) + review_receipt_id: str | None = None + +class CouncilAuthorityError(ValueError): + """Raised when Council attempts to exercise authority outside REVIEW.""" -def create_council(root: Path, artifact_hash: str, budget_hash: str, seats: list[CouncilSeat], *, task_id: str | None = None) -> CouncilState: + +def _row(value: ChallengeRequest | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(value, ChallengeRequest): + return json.loads(json.dumps(value, default=lambda item: item.__dict__, sort_keys=True)) + return dict(value) + + +def derive_challenge_seats( + challenges: Sequence[ChallengeRequest | Mapping[str, Any]], + *, + max_seats: int = 6, + reviewer_provenance: str | None = None, +) -> list[CouncilSeat]: + """Derive deterministic residual-gap seats from open neutral challenges. + + One seat is created per bound challenge until ``max_seats``. Resolved and + dismissed challenges are ignored. Seat count never establishes independence. + """ + if not 3 <= max_seats <= 6: + raise ValueError("max_seats must be between 3 and 6") + rows = [ + _row(challenge) + for challenge in challenges + if str(_row(challenge).get("state") or "PROPOSED") not in _TERMINAL_CHALLENGE_STATES + ] + rows.sort( + key=lambda item: ( + not bool(item.get("load_bearing")), + -int(item.get("risk_rank") or 0), + -int(item.get("information_rank") or 0), + int(item.get("cost_rank") or 0), + str(item.get("challenge_id") or ""), + ) + ) + seats: list[CouncilSeat] = [] + for row in rows[:max_seats]: + challenge_kind = _kind(row.get("kind")) + if challenge_kind is None: + continue + role, method, fallback_discriminator, fallback_capability = _RESIDUAL_SEAT_CLASSES[challenge_kind] + selected_plan = row.get("selected_plan") if isinstance(row.get("selected_plan"), dict) else {} + discriminator = str( + selected_plan.get("action") + or row.get("refuter") + or fallback_discriminator + ) + challenge_id = str(row.get("challenge_id") or digest(row)[:16]) + evidence_partition = f"challenge:{challenge_id}" + seats.append( + CouncilSeat( + seat_id=f"seat-{digest({'challenge_id': challenge_id, 'role': role})[:12]}", + role=role, + question=str(row.get("hypothesis") or f"Resolve {challenge_kind.value}"), + method=method, + evidence_partition=evidence_partition, + challenge_kind=challenge_kind, + discriminator=discriminator, + required_capability=( + str(row.get("required_capability")) + if row.get("required_capability") + else fallback_capability + ), + target_obligation_id=str(row.get("obligation_id") or ""), + refuter=str(row.get("refuter") or discriminator), + reviewer_provenance=reviewer_provenance, + challenge_contract="vnext", + ) + ) + if seats and not any("skeptic" in seat.role.lower() or "adversarial" in seat.role.lower() for seat in seats): + first = seats[0] + seats[0] = CouncilSeat( + seat_id=first.seat_id, + role=f"skeptic / {first.role}", + question=first.question, + method=first.method, + evidence_partition=first.evidence_partition, + challenge_kind=first.challenge_kind, + discriminator=first.discriminator, + required_capability=first.required_capability, + target_obligation_id=first.target_obligation_id, + refuter=first.refuter, + reviewer_provenance=first.reviewer_provenance, + challenge_contract="vnext", + ) + return seats + + +def _distinct_partition(left: CouncilSeat, right: CouncilSeat) -> bool: + return bool( + left.evidence_partition + and right.evidence_partition + and _norm(left.evidence_partition) != _norm(right.evidence_partition) + ) + + +def _validate_challenge_seats(seats: Sequence[CouncilSeat]) -> None: + vnext = [seat for seat in seats if seat.challenge_contract == "vnext"] + if vnext and len(vnext) != len(seats): + raise ValueError("Council cannot mix legacy and vNext challenge seats") + if not vnext: + return + for index, left in enumerate(seats): + for right in seats[index + 1 :]: + duplicate_kind = left.challenge_kind == right.challenge_kind + duplicate_discriminator = _norm(left.discriminator) == _norm(right.discriminator) + if (duplicate_kind or duplicate_discriminator) and not _distinct_partition(left, right): + raise ValueError( + "duplicate Council challenge kind/discriminator requires explicit distinct evidence partitions" + ) + + +def create_council( + root: Path, + artifact_hash: str, + budget_hash: str, + seats: list[CouncilSeat], + *, + task_id: str | None = None, +) -> CouncilState: if not 3 <= len(seats) <= 6: raise ValueError("Council requires 3-6 seats") - if len({s.seat_id for s in seats}) != len(seats): + if len({seat.seat_id for seat in seats}) != len(seats): raise ValueError("Council seat IDs must be unique") - if len({s.question for s in seats}) != len(seats): + if len({_norm(seat.question) for seat in seats}) != len(seats): raise ValueError("each Council seat must own a distinct question") - if not any("skeptic" in s.role.lower() or "adversarial" in s.role.lower() for s in seats): + if not any("skeptic" in seat.role.lower() or "adversarial" in seat.role.lower() for seat in seats): raise ValueError("Council requires at least one skeptic/adversarial seat") if not artifact_hash.strip(): raise ValueError("Council requires a frozen artifact hash") if not budget_hash.strip(): raise ValueError("Council requires a frozen total-budget hash") + _validate_challenge_seats(seats) state = CouncilState(new_id("council"), task_id, artifact_hash, budget_hash, seats) RuntimeStore(root).write_named_state("councils", state.council_id, _plain_state(state)) return state def _plain_state(state: CouncilState) -> dict[str, Any]: - return json.loads(json.dumps(state, default=lambda o: o.__dict__, sort_keys=True)) + return json.loads(json.dumps(state, default=lambda value: value.__dict__, sort_keys=True)) def _load(root: Path, council_id: str) -> CouncilState: raw = RuntimeStore(root).read_named_state("councils", council_id) if raw is None: raise KeyError(council_id) - raw["seats"] = [CouncilSeat(**s) for s in raw["seats"]] + raw["seats"] = [CouncilSeat(**seat) for seat in raw["seats"]] return CouncilState(**raw) def commitment_digest(seat_id: str, nonce: str, submission: SeatSubmission) -> str: - """Hiding + binding commitment over one seat's submission. - - The nonce is what makes it hiding: without it the commitment is a digest over a - low-entropy structured value, so an observer can confirm a guessed submission, - and two seats submitting identical text produce identical commitments. - """ + """Hiding + binding commitment over one seat's submission.""" return digest({"seat_id": seat_id, "nonce": nonce, "submission": submission}) @@ -118,16 +515,12 @@ def commit( *, nonce: str | None = None, ) -> tuple[str, str]: - """Record a seat commitment. Returns (commitment, nonce). - - The submission itself is NOT stored here: storing it at commit time would defeat - the point of committing, since the sealed copy is readable before any reveal. - """ + """Record a seat commitment. Returns ``(commitment, nonce)``.""" store = RuntimeStore(root) state = _load(root, council_id) if state.phase != "COMMIT": raise ValueError("Council is not accepting initial commits") - if seat_id not in {s.seat_id for s in state.seats}: + if seat_id not in {seat.seat_id for seat in state.seats}: raise KeyError(seat_id) if seat_id in state.commitments: raise ValueError("seat already committed") @@ -157,7 +550,7 @@ def reveal( if commitment_digest(seat_id, nonce, submission) != expected: return False state.sealed[seat_id] = {"commitment": expected} - state.revealed[seat_id] = json.loads(json.dumps(submission, default=lambda o: o.__dict__)) + state.revealed[seat_id] = json.loads(json.dumps(submission, default=lambda value: value.__dict__)) if len(state.revealed) == len(state.seats): state.phase = "CROSS_CRITIQUE" store.write_named_state("councils", council_id, _plain_state(state)) @@ -169,52 +562,126 @@ def record_cross_critique(root: Path, council_id: str, critique: CrossCritique) state = _load(root, council_id) if state.phase != "CROSS_CRITIQUE": raise ValueError("cross-critique is allowed only after all reveals") - seat_ids = {s.seat_id for s in state.seats} + seat_ids = {seat.seat_id for seat in state.seats} if critique.critic_seat_id not in seat_ids or critique.target_seat_id not in seat_ids: raise KeyError("unknown Council seat in cross-critique") - if any(row.get("critic_seat_id") == critique.critic_seat_id and row.get("target_seat_id") == critique.target_seat_id for row in state.cross_critiques): + if any( + row.get("critic_seat_id") == critique.critic_seat_id + and row.get("target_seat_id") == critique.target_seat_id + for row in state.cross_critiques + ): raise ValueError("duplicate critic-target cross-critique") - state.cross_critiques.append(json.loads(json.dumps(critique, default=lambda o: o.__dict__))) + state.cross_critiques.append( + json.loads(json.dumps(critique, default=lambda value: value.__dict__)) + ) store.write_named_state("councils", council_id, _plain_state(state)) -def _jaccard(a: set[str], b: set[str]) -> float: - if not a and not b: +def record_finding(root: Path, council_id: str, finding: CouncilFinding) -> None: + """Register a structured finding after reveal and before synthesis.""" + store = RuntimeStore(root) + state = _load(root, council_id) + if state.phase != "CROSS_CRITIQUE": + raise ValueError("Council findings are recorded only after all reveals") + if finding.finding_id in state.findings: + raise ValueError("duplicate Council finding_id") + seat_map = {seat.seat_id: seat for seat in state.seats} + seat = seat_map.get(finding.seat_id) + if seat is None: + raise KeyError(finding.seat_id) + revealed = state.revealed.get(finding.seat_id, {}) + if finding.finding_id not in revealed.get("findings", []): + raise ValueError("structured finding must be named in the seat's revealed submission") + if seat.challenge_contract == "vnext": + if finding.challenge_kind != seat.challenge_kind: + raise ValueError("finding challenge_kind does not match its Council seat") + if finding.target_obligation_id != seat.target_obligation_id: + raise ValueError("finding target_obligation_id does not match its Council seat") + if seat.required_capability and finding.required_capability != seat.required_capability: + raise ValueError("finding required_capability does not match its Council seat") + if _norm(finding.refuter) != _norm(seat.refuter): + raise ValueError("finding refuter does not match its Council seat") + if _norm(finding.evidence_partition) != _norm(seat.evidence_partition): + raise ValueError("finding evidence_partition does not match its Council seat") + state.findings[finding.finding_id] = json.loads( + json.dumps(finding, default=lambda value: value.__dict__, sort_keys=True) + ) + store.write_named_state("councils", council_id, _plain_state(state)) + + +def _jaccard(left: set[str], right: set[str]) -> float: + if not left and not right: return 1.0 - union = a | b - return len(a & b) / len(union) if union else 0.0 + union = left | right + return len(left & right) / len(union) if union else 0.0 def overlap_matrix(state: CouncilState) -> dict[str, Any]: if not state.revealed: - return {"pairs": [], "diagnostics": {"seat_count": len(state.seats), "distinct_methods": len({s.method for s in state.seats})}} - pairs = [] + return { + "pairs": [], + "diagnostics": { + "seat_count": len(state.seats), + "distinct_methods": len({seat.method for seat in state.seats}), + "independence_status": "NOT_ESTABLISHED_BY_SEAT_COUNT", + }, + } + pairs: list[dict[str, Any]] = [] ids = sorted(state.revealed) - seat_map = {s.seat_id: s for s in state.seats} - for i, left in enumerate(ids): - for right in ids[i + 1 :]: + seat_map = {seat.seat_id: seat for seat in state.seats} + for index, left in enumerate(ids): + for right in ids[index + 1 :]: a = state.revealed[left] b = state.revealed[right] - pairs.append({ - "left": left, - "right": right, - "evidence_overlap": _jaccard(set(a.get("evidence_ids", [])), set(b.get("evidence_ids", []))), - "provenance_overlap": _jaccard(set(a.get("provenance_groups", [])), set(b.get("provenance_groups", []))), - "method_same": seat_map[left].method == seat_map[right].method, - "finding_overlap": _jaccard(set(a.get("findings", [])), set(b.get("findings", []))), - }) + left_seat = seat_map[left] + right_seat = seat_map[right] + same_reviewer = bool( + left_seat.reviewer_provenance + and right_seat.reviewer_provenance + and left_seat.reviewer_provenance == right_seat.reviewer_provenance + ) + pairs.append( + { + "left": left, + "right": right, + "evidence_overlap": _jaccard( + set(a.get("evidence_ids", [])), + set(b.get("evidence_ids", [])), + ), + "provenance_overlap": _jaccard( + set(a.get("provenance_groups", [])), + set(b.get("provenance_groups", [])), + ), + "method_same": left_seat.method == right_seat.method, + "reviewer_provenance_same": same_reviewer, + "finding_overlap": _jaccard( + set(a.get("findings", [])), + set(b.get("findings", [])), + ), + "independence_status": "NOT_ESTABLISHED", + } + ) + def mean(key: str) -> float | None: values = [float(row[key]) for row in pairs] return sum(values) / len(values) if values else None + return { "pairs": pairs, "diagnostics": { "seat_count": len(state.seats), - "distinct_methods": len({s.method for s in state.seats}), + "distinct_methods": len({seat.method for seat in state.seats}), + "same_reviewer_provenance_pairs": sum( + bool(row["reviewer_provenance_same"]) for row in pairs + ), "mean_evidence_overlap": mean("evidence_overlap"), "mean_provenance_overlap": mean("provenance_overlap"), "mean_finding_overlap": mean("finding_overlap"), - "interpretation": "Overlap is a correlation/common-cause diagnostic, not a proof of statistical independence.", + "independence_status": "NOT_ESTABLISHED_BY_SEAT_COUNT", + "interpretation": ( + "Overlap and reviewer provenance are common-cause diagnostics; " + "nominal seat count never establishes statistical independence." + ), }, } @@ -225,42 +692,74 @@ def brier_score(confidence: float, outcome_correct: bool) -> float: return (confidence - (1.0 if outcome_correct else 0.0)) ** 2 -def record_control(root: Path, obligation_id: str, *, artifact_hash: str, budget_hash: str, kind: str, output_hash: str, verdict: Verdict, verifier: str) -> Receipt: +def record_control( + root: Path, + obligation_id: str, + *, + artifact_hash: str, + budget_hash: str, + kind: str, + output_hash: str, + verdict: Verdict, + verifier: str, +) -> Receipt: kind = kind.upper() if kind not in {"DIRECT", "VOTE"}: raise ValueError("Council control kind must be DIRECT or VOTE") receipt = Receipt( - receipt_id=new_id("rcpt"), module="council", obligation_id=obligation_id, - verdict=verdict, action=f"control:{kind.lower()}", - input_hash=digest({"artifact_hash": artifact_hash, "budget_hash": budget_hash, "kind": kind}), + receipt_id=new_id("rcpt"), + module="council", + obligation_id=obligation_id, + verdict=verdict, + action=f"control:{kind.lower()}", + input_hash=digest( + {"artifact_hash": artifact_hash, "budget_hash": budget_hash, "kind": kind} + ), output_hash=output_hash, - evidence=(EvidenceRef( - evidence_class=EvidenceClass.MEASURED, - verifier=verifier, - metadata={"control_kind": kind, "artifact_hash": artifact_hash, "budget_hash": budget_hash}, - ),), - verifier=verifier, started_at=utcnow(), finished_at=utcnow(), - notes="Control receipt records comparability metadata; it does not itself establish Council benefit.", + evidence=( + EvidenceRef( + evidence_class=EvidenceClass.MEASURED, + verifier=verifier, + metadata={ + "control_kind": kind, + "artifact_hash": artifact_hash, + "budget_hash": budget_hash, + }, + ), + ), + verifier=verifier, + started_at=utcnow(), + finished_at=utcnow(), + notes=( + "Control receipt records comparability metadata; it does not itself " + "establish Council benefit." + ), ) RuntimeStore(root).write_receipt(receipt) return receipt -def _matched_control(store: RuntimeStore, receipt_id: str | None, kind: str, artifact_hash: str, budget_hash: str, obligation_id: str | None = None) -> bool: +def _matched_control( + store: RuntimeStore, + receipt_id: str | None, + kind: str, + artifact_hash: str, + budget_hash: str, + obligation_id: str | None = None, +) -> bool: if not receipt_id: return False receipt = store.read_receipt(receipt_id) if not receipt or receipt.get("module") != "council": return False - # A control run for a different obligation is not this obligation's control. if obligation_id is not None and receipt.get("obligation_id") != obligation_id: return False for evidence in receipt.get("evidence", []): - meta = evidence.get("metadata", {}) if isinstance(evidence, dict) else {} + metadata = evidence.get("metadata", {}) if isinstance(evidence, dict) else {} if ( - meta.get("control_kind") == kind - and meta.get("artifact_hash") == artifact_hash - and meta.get("budget_hash") == budget_hash + metadata.get("control_kind") == kind + and metadata.get("artifact_hash") == artifact_hash + and metadata.get("budget_hash") == budget_hash ): return receipt.get("verdict") == Verdict.CLEARED.value return False @@ -271,6 +770,32 @@ def _cross_critique_complete(state: CouncilState) -> bool: return critics == {seat.seat_id for seat in state.seats} +def _obligation_row(store: RuntimeStore, state: CouncilState, obligation_id: str) -> dict[str, Any] | None: + task_id = state.task_id or store.task_for_obligation(obligation_id) + if task_id is None: + return None + task = store.read_task(task_id) + if task is None: + raise CouncilAuthorityError("Council task is missing or corrupt") + for obligation in task.get("obligations", []): + if obligation.get("obligation_id") == obligation_id: + return obligation + raise CouncilAuthorityError("Council receipt obligation is not part of the bound task") + + +def _require_review_authority(store: RuntimeStore, state: CouncilState, obligation_id: str) -> None: + obligation = _obligation_row(store, state, obligation_id) + if obligation is None: + return + if obligation.get("kind") != ObligationKind.REVIEW.value: + raise CouncilAuthorityError( + "Council can clear only a REVIEW obligation, never the target domain obligation" + ) + required = obligation.get("required_module") + if required not in (None, "council"): + raise CouncilAuthorityError("REVIEW obligation is bound to a different module") + + def finalize( root: Path, council_id: str, @@ -286,8 +811,24 @@ def finalize( state = _load(root, council_id) if state.phase == "CLOSED": raise ValueError("Council is already CLOSED; finalize is not re-callable") - direct_exists = _matched_control(store, direct_control_receipt, "DIRECT", state.artifact_hash, state.budget_hash, obligation_id) - vote_exists = _matched_control(store, vote_control_receipt, "VOTE", state.artifact_hash, state.budget_hash, obligation_id) + _require_review_authority(store, state, obligation_id) + + direct_exists = _matched_control( + store, + direct_control_receipt, + "DIRECT", + state.artifact_hash, + state.budget_hash, + obligation_id, + ) + vote_exists = _matched_control( + store, + vote_control_receipt, + "VOTE", + state.artifact_hash, + state.budget_hash, + obligation_id, + ) reveal_complete = len(state.revealed) == len(state.seats) critique_complete = _cross_critique_complete(state) missing: list[str] = list(unresolved or []) @@ -297,13 +838,14 @@ def finalize( missing.append("cross-critique-participation-incomplete") if not direct_exists: missing.append("same-artifact-same-budget-direct-control-missing-invalid-or-mismatched") + if all(seat.challenge_contract == "vnext" for seat in state.seats): + for finding_id in supported_findings: + if finding_id not in state.findings: + missing.append(f"supported-finding-missing-structured-record:{finding_id}") if missing: verdict = Verdict.UNKNOWN else: - # This clears the REVIEW obligation only: the panel executed its frozen - # method and produced supported finding IDs under a valid control. It does - # not imply that Council is globally superior to DIRECT or VOTE. verdict = Verdict.CLEARED if supported_findings else Verdict.UNKNOWN if not supported_findings: missing.append("no-supported-finding-after-synthesis") @@ -311,32 +853,145 @@ def finalize( state.phase = "CLOSED" state.direct_control_receipt = direct_control_receipt state.vote_control_receipt = vote_control_receipt - store.write_named_state("councils", council_id, _plain_state(state)) + state.supported_finding_ids = list(dict.fromkeys(supported_findings)) matrix = overlap_matrix(state) receipt = Receipt( - receipt_id=new_id("rcpt"), module="council", obligation_id=obligation_id, - verdict=verdict, action="commit-reveal-controlled-evidence-review", - input_hash=digest({"artifact_hash": state.artifact_hash, "budget_hash": state.budget_hash, "commitments": state.commitments}), + receipt_id=new_id("rcpt"), + module="council", + obligation_id=obligation_id, + verdict=verdict, + action="commit-reveal-controlled-evidence-review", + input_hash=digest( + { + "artifact_hash": state.artifact_hash, + "budget_hash": state.budget_hash, + "commitments": state.commitments, + } + ), output_hash=synthesis_hash, - evidence=(EvidenceRef( - evidence_class=EvidenceClass.DERIVED, - verifier="council_runtime", - metadata={ - "overlap": matrix, - "confidence_status": "UNCALIBRATED_UNLESS_PROSPECTIVELY_SCORED", - "direct_control_matched": direct_exists, - "vote_control_matched": vote_exists, - "cross_critique_complete": critique_complete, - }, - ),), - verifier="council_runtime", started_at=utcnow(), finished_at=utcnow(), + evidence=( + EvidenceRef( + evidence_class=EvidenceClass.DERIVED, + verifier="council_runtime", + metadata={ + "overlap": matrix, + "confidence_status": "UNCALIBRATED_UNLESS_PROSPECTIVELY_SCORED", + "direct_control_matched": direct_exists, + "vote_control_matched": vote_exists, + "cross_critique_complete": critique_complete, + "authority": "REVIEW_ONLY", + "target_domain_clearance_authorized": False, + }, + ), + ), + verifier="council_runtime", + started_at=utcnow(), + finished_at=utcnow(), unresolved=tuple(dict.fromkeys(missing)), - notes=json.dumps({ - "supported_finding_hashes": [digest(finding) for finding in supported_findings], - "overlap_diagnostics": matrix.get("diagnostics", {}), - "boundary": "CLEARED means the frozen Council review protocol completed; it is not a claim that Council outperforms controls.", - }, sort_keys=True), + notes=json.dumps( + { + "supported_finding_hashes": [ + digest(finding) for finding in supported_findings + ], + "overlap_diagnostics": matrix.get("diagnostics", {}), + "boundary": ( + "CLEARED means the frozen Council REVIEW protocol completed. " + "Council findings are proposal-only and cannot clear target domain obligations." + ), + }, + sort_keys=True, + ), task_id=state.task_id, ) + state.review_receipt_id = receipt.receipt_id + store.write_named_state("councils", council_id, _plain_state(state)) store.write_receipt(receipt) return receipt + + +def _validate_target_binding( + store: RuntimeStore, + task_id: str, + finding: CouncilFinding, +) -> None: + task = store.read_task(task_id) + if task is None: + raise CouncilAuthorityError("Council target task is missing or corrupt") + for obligation in task.get("obligations", []): + if obligation.get("obligation_id") != finding.target_obligation_id: + continue + required = obligation.get("required_module") + if required and required != finding.target_module: + raise CouncilAuthorityError( + "Council finding target_module does not match the target obligation" + ) + return + raise CouncilAuthorityError("Council finding targets an obligation outside the bound task") + + +def propose_supported_finding_challenge( + root: Path, + council_id: str, + finding_id: str, + *, + proposer: str = "council_runtime", +) -> ChallengeRequest: + """Convert one supported structured finding into a neutral proposal. + + The returned ``ChallengeRequest`` has ``origin=COUNCIL`` and proposal-only + authority. This function never writes a Council receipt for the target obligation + and never resolves the emitted challenge. + """ + store = RuntimeStore(root) + state = _load(root, council_id) + if state.phase != "CLOSED": + raise ChallengeError("Council must be finalized before emitting a supported challenge") + if finding_id not in state.supported_finding_ids: + raise ChallengeError("Council finding was not supported by final synthesis") + raw = state.findings.get(finding_id) + if raw is None: + raise ChallengeError("supported finding lacks a structured challenge record") + finding = CouncilFinding(**raw) + if state.task_id is None: + raise ChallengeError("Council challenge emission requires a bound task_id") + _validate_target_binding(store, state.task_id, finding) + + seat = next( + (candidate for candidate in state.seats if candidate.seat_id == finding.seat_id), + None, + ) + if seat is None: + raise ChallengeError("supported finding references an unknown Council seat") + + request = ChallengeRequest( + challenge_id=new_id("chal"), + task_id=state.task_id, + obligation_id=finding.target_obligation_id, + target_module=finding.target_module, + origin=ChallengeOrigin.COUNCIL, + kind=finding.challenge_kind, + hypothesis=finding.hypothesis, + alternative=finding.alternative, + refuter=finding.refuter, + consequence_if_true=finding.consequence_if_true, + load_bearing=finding.load_bearing, + required_capability=finding.required_capability, + candidate_hash=finding.candidate_hash, + scope_hash=finding.scope_hash, + obligation_set_hash=finding.obligation_set_hash, + proposer=proposer, + proposer_provenance=finding.proposer_provenance or state.review_receipt_id, + metadata={ + **finding.metadata, + "council_id": council_id, + "review_receipt_id": state.review_receipt_id, + "finding_id": finding.finding_id, + "seat_id": finding.seat_id, + "seat_discriminator": seat.discriminator, + "evidence_partition": finding.evidence_partition, + "authority": "PROPOSAL_ONLY", + "domain_clearance_authorized": False, + }, + ) + propose_challenge(root, request) + return request