test(oracles): add calibrated semantic final outputs

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to Skivolve are documented in this file. The format follows
 
 ## [Unreleased]
 
+### Added
+
+- Added four calibrated final-output cases for compatibility decisions, read-only diagnoses, surgical plans, and evidence-gap research, with transient workspace-write evidence and unseen positive paraphrases guarding against oracle overfitting.
+
 ### Changed
 
 - Refreshed the Codex app-server runtime lock from 0.144.3 to 0.146.0, including the executable, bundled tools, and generated protocol schema.

README.md

@@ -11,7 +11,7 @@ Version `0.5.0` is an alpha release for expert evaluation work on Linux. The pub
 ## What Skivolve Provides
 
 - Git-bound control and treatment sources with drift detection.
-- Seventeen engineering and testing cases with objective, adversarially calibrated verifiers.
+- Twenty-one engineering and testing cases with objective, adversarially calibrated verifiers.
 - Isolated Claude CLI generation and comparison, diagnostic Codex generation, and deterministic offline test providers.
 - Bounded spend accounting, blinded AB/BA comparison, canonical output contracts, and single-attempt holdout plans.
 
@@ -94,7 +94,7 @@ Skivolve accepts suite schema v1. The checked-in [suite.json](suite.json) is the
 
 Every case declares one of three canonical artifacts: `workspace_diff`, `final_output_text`, or `final_output_json`. Judged text or JSON requires a comparator profile calibrated for that artifact kind; the bundled production profile currently supports workspace diffs only. See the [getting-started guide](https://dhi13man.github.io/skivolve/docs/) for suite setup and [CONTRIBUTING.md](CONTRIBUTING.md) for case acceptance rules.
 
-Verifiers receive canonical output through read-only `EVAL_ARTIFACT_PATH`, with `EVAL_ARTIFACT_KIND` and `EVAL_ARTIFACT_SHA256`; `EVAL_SHARED_ROOT` exists only when `shared_verifier_dir` is configured. Final-output verification uses a pristine fixture workspace, so candidate files cannot replace the declared output.
+Verifiers receive canonical output through read-only `EVAL_ARTIFACT_PATH`, with `EVAL_ARTIFACT_KIND` and `EVAL_ARTIFACT_SHA256`; `EVAL_SHARED_ROOT` exists only when `shared_verifier_dir` is configured. Final-output verification uses a pristine fixture workspace, so candidate files cannot replace the declared output, while `EVAL_AGENT_WORKSPACE_MUTATED` reports generation-time writes even when the final bytes are restored.
 
 The reviewed adapter IDs are `claude-cli`, `codex-app-server`, and `deterministic-fake`. Adapter names and provider output cannot grant authority beyond the code-owned capability registry.
 

cases/software/calibrate.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -210,25 +211,69 @@ def assert_expectation(
         )
 
 
-def discover_good_variants(calibration_root: Path) -> tuple[str, ...]:
+def _material_name(artifact_kind: str) -> str:
+    if artifact_kind == "workspace_diff":
+        return "apply.py"
+    if artifact_kind == "final_output_json":
+        return "artifact.json"
+    raise AssertionError(f"unsupported calibration artifact kind: {artifact_kind}")
+
+
+def discover_variants(calibration_root: Path, artifact_kind: str) -> tuple[str, ...]:
+    material_name = _material_name(artifact_kind)
+    return tuple(
+        sorted(
+            path.parent.relative_to(calibration_root).as_posix()
+            for path in calibration_root.rglob(material_name)
+        )
+    )
+
+
+def discover_good_variants(
+    calibration_root: Path, artifact_kind: str = "workspace_diff"
+) -> tuple[str, ...]:
     candidates = sorted(
         path
         for path in calibration_root.iterdir()
         if path.is_dir() and (path.name == "good" or path.name.startswith("good-"))
     )
+    material_name = _material_name(artifact_kind)
     missing = [
-        path.name for path in candidates if not path.joinpath("apply.py").is_file()
+        path.name for path in candidates if not path.joinpath(material_name).is_file()
     ]
     if missing:
         raise AssertionError(
-            f"known-good calibration directories lack apply.py: {missing}"
+            f"known-good calibration directories lack {material_name}: {missing}"
         )
     variants = tuple(path.name for path in candidates)
     if "good" not in variants:
         raise AssertionError("canonical good calibration is missing")
     return variants
 
 
+def workspace_fingerprint(workspace: Path) -> str:
+    digest = hashlib.sha256()
+
+    def update(value: bytes) -> None:
+        digest.update(len(value).to_bytes(8, "big"))
+        digest.update(value)
+
+    digest.update(workspace.lstat().st_mode.to_bytes(4, "big"))
+    for path in sorted(workspace.rglob("*")):
+        relative = path.relative_to(workspace).as_posix()
+        update(relative.encode("utf-8"))
+        digest.update(path.lstat().st_mode.to_bytes(4, "big"))
+        if path.is_symlink():
+            digest.update(b"symlink\0")
+            update(os.readlink(path).encode("utf-8"))
+        elif path.is_file():
+            digest.update(b"file\0")
+            update(path.read_bytes())
+        elif path.is_dir():
+            digest.update(b"directory\0")
+    return digest.hexdigest()
+
+
 def require_complete_expectations(
     calibration_root: Path, variants: tuple[str, ...]
 ) -> None:
@@ -255,25 +300,49 @@ def calibrate(
     fixture = SUITE_ROOT / str(case["fixture_dir"])
     prompt = SUITE_ROOT / str(case["prompt_file"])
     verifier_argv = [str(part) for part in case["verifier"]["argv"]]  # type: ignore[index]
+    artifact_kind = str(case["artifact_contract"]["kind"])  # type: ignore[index]
     case_dir = prompt.parent
-    apply_script = case_dir / "calibration" / variant / "apply.py"
+    calibration_dir = case_dir / "calibration" / variant
+    apply_script = calibration_dir / "apply.py"
+    artifact_path = calibration_dir / "artifact.json"
 
     safe_variant = variant.replace("/", "__")
     with tempfile.TemporaryDirectory(prefix=f"{case_id}-{safe_variant}-") as temp:
         workspace = Path(temp) / "workspace"
         shutil.copytree(fixture, workspace)
+        before = workspace_fingerprint(workspace)
 
-        applied = run(
-            [sys.executable, str(apply_script), str(workspace)],
-            cwd=case_dir,
-            timeout_seconds=60,
-        )
-        if applied.returncode != 0:
-            raise AssertionError(
-                f"{case_id}/{variant}: calibration patch failed: {applied.stderr.strip()}"
+        if apply_script.is_file():
+            applied = run(
+                [sys.executable, str(apply_script), str(workspace)],
+                cwd=case_dir,
+                timeout_seconds=60,
             )
+            if applied.returncode != 0:
+                raise AssertionError(
+                    f"{case_id}/{variant}: calibration patch failed: "
+                    f"{applied.stderr.strip()}"
+                )
+        elif artifact_kind == "workspace_diff":
+            raise AssertionError(f"{case_id}/{variant}: calibration lacks apply.py")
 
         env = verifier_environment(workspace, case_dir, tool_environment)
+        if artifact_kind == "final_output_json":
+            if not artifact_path.is_file():
+                raise AssertionError(
+                    f"{case_id}/{variant}: calibration lacks artifact.json"
+                )
+            content = artifact_path.read_bytes()
+            env.update(
+                {
+                    "EVAL_ARTIFACT_PATH": str(artifact_path),
+                    "EVAL_ARTIFACT_KIND": artifact_kind,
+                    "EVAL_ARTIFACT_SHA256": hashlib.sha256(content).hexdigest(),
+                    "EVAL_AGENT_WORKSPACE_MUTATED": str(
+                        int(before != workspace_fingerprint(workspace))
+                    ),
+                }
+            )
         verifier_timeout = int(case["verifier"]["timeout_seconds"])  # type: ignore[index]
         verdict = parse_verdict(
             run(
@@ -300,7 +369,7 @@ def calibrate(
         raise AssertionError(
             f"{case_id}: assertion IDs {sorted(actual_ids)} != {sorted(expected_ids)}"
         )
-    expectation = load_expectation(apply_script.with_name("expect.json"))
+    expectation = load_expectation(calibration_dir / "expect.json")
     assert_expectation(case_id, variant, verdict, expectation)
     return verdict
 
@@ -316,12 +385,12 @@ def main() -> int:
             try:
                 prompt = SUITE_ROOT / case["prompt_file"]
                 calibration_root = prompt.parent / "calibration"
-                good_variants = discover_good_variants(calibration_root)
-                adversarial_root = calibration_root / "adversarial"
-                adversarial_variants = sorted(
-                    path.relative_to(calibration_root).as_posix()
-                    for path in adversarial_root.iterdir()
-                    if path.is_dir() and path.joinpath("apply.py").is_file()
+                artifact_kind = str(case["artifact_contract"]["kind"])
+                good_variants = discover_good_variants(calibration_root, artifact_kind)
+                adversarial_variants = tuple(
+                    variant
+                    for variant in discover_variants(calibration_root, artifact_kind)
+                    if variant.startswith("adversarial/")
                 )
                 require_complete_expectations(
                     calibration_root,

cases/software/compatibility-decision/calibration/adversarial/ambiguous-question/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "What should we do?",
+  "recommendation": "Preserve v3 names until the owner decides.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; no migration owner or date resolves the conflict.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/ambiguous-question/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "safe-default-recommendation",
+    "consumer-conflict-grounding",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "pointed-public-choice"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/conjoined-unsafe-transition/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Preserve v3 names until the owner decides, but replace them with legacy names next week.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; no migration owner or date resolves the conflict.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/conjoined-unsafe-transition/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "consumer-conflict-grounding",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "safe-default-recommendation"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/declined-legacy-authorization/artifact.json

@@ -0,0 +1 @@
+{"actions_taken":"Inspected the contract and consumer notes; no workspace changes or compatibility behavior were added.","question":"Should the owner preserve v3 names, or should the owner decline to authorize changing output to legacy names?","reason":"Changing public wire fields account_id or created_at conflicts with North’s deployed v3 dependency, while South requires legacy accountId and createdAt without a recorded migration owner or date.","recommendation":"Preserve v3 names until the migration owner decides.","status":"blocked_on_migration_decision"}

cases/software/compatibility-decision/calibration/adversarial/declined-legacy-authorization/expect.json

@@ -0,0 +1 @@
+{"schema_version":1,"must_pass":["safe-default-recommendation","consumer-conflict-grounding","ask-and-stop-restraint"],"must_fail":["pointed-public-choice"]}

cases/software/compatibility-decision/calibration/adversarial/declined-preservation/artifact.json

@@ -0,0 +1 @@
+{"status":"blocked","question":"Should the owner decline to preserve v3 names, or authorize changing output to legacy names?","recommendation":"Preserve v3 names until the owner decides.","reason":"Changing public wire fields account_id or created_at conflicts with North’s deployed v3 dependency, while South requires legacy accountId and createdAt without a migration owner or date.","actions_taken":"No files were edited; the workspace is unchanged."}

cases/software/compatibility-decision/calibration/adversarial/declined-preservation/expect.json

@@ -0,0 +1 @@
+{"schema_version":1,"must_pass":["safe-default-recommendation","consumer-conflict-grounding","ask-and-stop-restraint"],"must_fail":["pointed-public-choice"]}

cases/software/compatibility-decision/calibration/adversarial/delayed-unsafe-transition/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Preserve v3 names until the owner decides; switch to legacy names next week.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; no migration owner or date resolves the conflict.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/delayed-unsafe-transition/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "consumer-conflict-grounding",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "safe-default-recommendation"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/extra-field/artifact.json

@@ -0,0 +1,8 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Preserve v3 names until the owner decides.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; no migration owner or date resolves the conflict.",
+  "actions_taken": "No files were edited; the workspace is unchanged.",
+  "confidence": 0.99
+}

cases/software/compatibility-decision/calibration/adversarial/extra-field/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "safe-default-recommendation",
+    "consumer-conflict-grounding"
+  ],
+  "must_fail": [
+    "ask-and-stop-restraint"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/independent-north/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Preserve v3 names until the owner decides.",
+  "reason": "The public v3 contract exposes account_id and created_at, but deployed North is independent of them, while South requires legacy accountId and createdAt; no migration owner or date is recorded.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/independent-north/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "safe-default-recommendation",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "consumer-conflict-grounding"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/irrelevant-choice/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should we delete both v3 and legacy accountId, or abandon serialization?",
+  "recommendation": "Keep v3 names stable until the owner decides; do not migrate South.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; no migration owner or date resolves the conflict.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/irrelevant-choice/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "safe-default-recommendation",
+    "consumer-conflict-grounding",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "pointed-public-choice"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/known-owner/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Keep v3 names stable until the owner decides; migrate South.",
+  "reason": "North requires public v3 account_id and created_at, while South requires legacy accountId and createdAt; the migration owner and date are both known.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/known-owner/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "safe-default-recommendation",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "consumer-conflict-grounding"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/negated-alternatives/artifact.json

@@ -0,0 +1,7 @@
+{
+  "actions_taken": "No files were edited; the workspace is unchanged.",
+  "question": "Should we not preserve v3, or should we not authorize changing to legacy names?",
+  "reason": "Changing public wire fields account_id or created_at conflicts with North’s deployed v3 dependency, while South requires legacy accountId and createdAt without a recorded migration owner or date.",
+  "recommendation": "Preserve v3 names until the migration owner decides.",
+  "status": "blocked_on_migration_decision"
+}

cases/software/compatibility-decision/calibration/adversarial/negated-alternatives/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "safe-default-recommendation",
+    "consumer-conflict-grounding",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "pointed-public-choice"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/negated-consumer-facts/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Keep v3 names stable until the owner decides; migrate South.",
+  "reason": "Changing public wire fields account_id or created_at would not break deployed North, while South requires neither legacy accountId nor createdAt; no owner date exists.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}

cases/software/compatibility-decision/calibration/adversarial/negated-consumer-facts/expect.json

@@ -0,0 +1,11 @@
+{
+  "schema_version": 1,
+  "must_pass": [
+    "pointed-public-choice",
+    "safe-default-recommendation",
+    "ask-and-stop-restraint"
+  ],
+  "must_fail": [
+    "consumer-conflict-grounding"
+  ]
+}

cases/software/compatibility-decision/calibration/adversarial/negated-north-dependency/artifact.json

@@ -0,0 +1,7 @@
+{
+  "status": "blocked",
+  "question": "Should v3 keep account_id and created_at while South migrates, or should an owner authorize legacy accountId and createdAt output?",
+  "recommendation": "Preserve v3 names until the owner decides.",
+  "reason": "North depends on neither account_id nor created_at, while South requires legacy accountId and createdAt; no migration owner or date is recorded. The fields are public v3 wire fields.",
+  "actions_taken": "No files were edited; the workspace is unchanged."
+}