From aaee2ac6d104e2621c9f4637b32e5d297b8d4b22 Mon Sep 17 00:00:00 2001 From: Eliauk Date: Thu, 16 Jul 2026 09:34:48 +0200 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20prune=20once=20per=20event=20?= =?UTF-8?q?=E2=80=94=20pruning=20inside=20the=20append=20loop=20strips=20s?= =?UTF-8?q?ibling=20groups=20on=20a=20shared=20event?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With two hooks registered on the same event (e.g. two PostToolUse/Bash entries), processing the second spec re-pruned the event list and silently dropped the group just added for the first. Latent with a single spec per event; bites as soon as a second one exists. Co-Authored-By: Claude Fable 5 --- install.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 9ce3131..bfc2efd 100755 --- a/install.sh +++ b/install.sh @@ -31,6 +31,7 @@ SPECS = [ ("SessionStart", None, "fable_profile_inject.py"), ("PreToolUse", "Agent|Task|Workflow", "fable_spawn_guard.py"), ("PostToolUse", "Bash", "fable_fail_streak.py"), + ("PostToolUse", "Bash", "fable_evidence_log.py"), ("Stop", None, "fable_close_guard.py"), ] NAMES = {fname for _, _, fname in SPECS} @@ -69,16 +70,19 @@ if mode == "uninstall": print("fable-mode hooks removed from %s" % settings_path) sys.exit(0) -# install: prune our old entries first (handles moves/upgrades), then add fresh. +# install: prune our old entries first (handles moves/upgrades), then add +# fresh. Prune ONCE per event before appending — pruning inside the append +# loop would strip the groups just added for an earlier spec on the same +# event (two hooks share PostToolUse/Bash). added = 0 +for event in {e for e, _, _ in SPECS}: + hooks[event] = prune(hooks.get(event, [])) for event, matcher, fname in SPECS: - entries = prune(hooks.get(event, [])) group = {"hooks": [{"type": "command", "command": "python3 %s" % os.path.join(hooks_dir, fname)}]} if matcher: group["matcher"] = matcher - entries.append(group) - hooks[event] = entries + hooks[event].append(group) added += 1 data["hooks"] = hooks From 9b950c8311c2bc5142b9cef44e2e2b040e796dc4 Mon Sep 17 00:00:00 2001 From: Eliauk Date: Thu, 16 Jul 2026 09:39:04 +0200 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20machine-written=20evidence=20?= =?UTF-8?q?=E2=80=94=20Evidence=20Logger=20hook,=20citation=20corroboratio?= =?UTF-8?q?n,=20opt-in=20acceptance=20replay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the honor-system gap in evidence-on-close. Today the Close Guard checks that an '-- evidence:' note EXISTS and is substantive — but the note is self-reported: a model can write 'evidence: pytest 21/21' without ever running pytest, and the guard passes. The most-needed check (did the acceptance actually run?) was the one thing the hooks couldn't see. Three pieces: 1. Evidence Logger (new PostToolUse/Bash hook, passive): appends every command's real outcome {cmd, exit, output tail} to .fable/evidence.jsonl. Evidence is machine-written from tool results, not typed by the model. Fabricating now requires a visible act (hand-editing the log) instead of a plausible sentence. Rotates at 512KB; records even while PAUSED. 2. Citation corroboration (Close Guard): a checked card that cites a `command` in its evidence note must have a successful run of that command in the log — never ran, or never exited 0, blocks the stop. Prose-only evidence (screenshots etc.) keeps the existing substantive- string rule; projects without a log (pre-logger) are untouched (fail-open). 3. Acceptance replay (opt-in, 'REPLAY: on' ledger line): before the round may end, re-run each cited acceptance — 'passed once' is not 'still passes'; a later card silently breaking an earlier one is caught at the door. Budgeted (30s/cmd via FABLE_REPLAY_TIMEOUT, 120s total) so a heavy suite can't hang the stop; off by default because replay costs real time. All existing invariants preserved: fail-open everywhere, .fable/ opt-in, loop-safe, stdlib only. Tests: tests/test_evidence.py (14 cases) + test_install updated for the fifth hook; existing suites green. Co-Authored-By: Claude Fable 5 --- SKILL.md | 8 +- hooks/README.md | 16 ++-- hooks/_fable_common.py | 138 ++++++++++++++++++++++++++++++++++ hooks/fable_close_guard.py | 97 +++++++++++++++++++++++- hooks/fable_evidence_log.py | 56 ++++++++++++++ hooks/fable_profile_inject.py | 6 +- templates/LEDGER.template.md | 6 ++ tests/test_evidence.py | 124 ++++++++++++++++++++++++++++++ tests/test_install.py | 8 +- 9 files changed, 447 insertions(+), 12 deletions(-) create mode 100644 hooks/fable_evidence_log.py create mode 100644 tests/test_evidence.py diff --git a/SKILL.md b/SKILL.md index daa87f7..44436e4 100644 --- a/SKILL.md +++ b/SKILL.md @@ -130,12 +130,13 @@ Selecting a profile: the user's words above, env `FABLE_ROUTING=quality|balanced ## Enforcement layer (hooks — mechanics in `hooks/README.md`) -Four hooks turn the most-shirked rules into hard blocks. Armed **per project** by a `.fable/` directory (searched upward, bounded at the git root); without it they pass through silently. Pressure applies **per round** via `.fable/LEDGER.md`: +Five hooks turn the most-shirked rules into hard blocks. Armed **per project** by a `.fable/` directory (searched upward, bounded at the git root); without it they pass through silently. Pressure applies **per round** via `.fable/LEDGER.md`: ``` - [ ] 1. card (machine-checkable acceptance) <- open: guards enforce -- [x] 2. done -- evidence: pytest 21/21 <- [x] REQUIRES a substantive evidence note +- [x] 2. done -- evidence: `pytest -q` 21/21 <- [x] REQUIRES substantive evidence; a cited `command` is checked against the machine-written evidence log - [~] 3. not this round -- deferred: reason +REPLAY: on <- optional: re-run cited acceptances before the round may end PAUSED: reason <- a line anywhere: enforcement off ``` @@ -145,7 +146,8 @@ missing.) - **Spawn Guard** (PreToolUse Agent/Task/Workflow): blocks a detailed spawn while the ledger has no **open** cards — no ledger, and equally a ledger holding only a finished round's closed cards (design gate: new fan-out needs a live card) — and blocks any spawn requesting a **model stronger than the session's** (model ceiling — checked on the `model` param and `model:` literals in Workflow scripts; stays active even when paused, it protects quota, not workflow). - **Fail-Streak Reminder** (PostToolUse Bash, advisory): every 3rd consecutive failing command injects the attribution ladder — stops grinding on the wrong layer mechanically, not by willpower. -- **Close Guard** (Stop): blocks ending the turn while open `- [ ]` items remain, **and** while any `- [x]` lacks an `-- evidence:` note (evidence-on-close: adjectives don't close cards). +- **Evidence Logger** (PostToolUse Bash, passive): appends every command's real outcome (command, exit code, output tail) to `.fable/evidence.jsonl` — the machine-written record the Close Guard checks citations against. Records even while paused; evidence gaps are worse than pauses. +- **Close Guard** (Stop): blocks ending the turn while open `- [ ]` items remain, while any `- [x]` lacks an `-- evidence:` note (evidence-on-close: adjectives don't close cards), while any cited evidence `command` has **no successful run in the evidence log** (machine corroboration: a citation that never ran is not evidence), and — with `REPLAY: on` — while any cited acceptance fails when **re-run now** ('passed once' is not 'still passes'). - **Profile Injector** (SessionStart): injects tier + routing + habits, **sized to the ledger state** — full when a round is starting/active, minimal when idle, one line when paused. **Wrap-up lint**: `python3 /hooks/fable_lint.py ` — machine-checks the discipline itself (SPEC source tags present, open cards name acceptance, closed cards carry evidence). Run it at step 7 of the execution template; findings are open work. diff --git a/hooks/README.md b/hooks/README.md index dfcd261..8cb113d 100644 --- a/hooks/README.md +++ b/hooks/README.md @@ -4,14 +4,15 @@ The enforcement layer: turn a few of fable-mode's prose rules into Claude Code hooks that actually block — ledger-before-delegation and close-verification, built around this repo's SPEC.md/PROGRESS.md conventions. -## Four hooks + one lint CLI +## Five hooks + one lint CLI | Hook | Event | What it does | |---|---|---| | `fable_profile_inject.py` | `SessionStart` | When the project has opted in, **auto-inject the tier by model + the six levers + ledger context recovery** (no need to type "use fable mode") | | `fable_spawn_guard.py` | `PreToolUse` (Agent\|Task\|Workflow) | When opted in: **block a detailed spawn with no ledger** (forces the plan gate) and **block any spawn requesting a model stronger than the session's** (the model ceiling) | | `fable_fail_streak.py` | `PostToolUse` (Bash) | Advisory, never blocks: at every 3rd **consecutive failing command**, inject the attribution ladder (harness → deployment → product; fix the class via an invariant). Streak state: `$TMPDIR/fable-mode-sessions/.fails`, reset on success. | -| `fable_close_guard.py` | `Stop` | While the ledger still has unchecked items, **block ending the turn** (cures early stopping / spinning). When all items are checked, **block if any `- [x]` lacks an evidence marker** (`-- evidence:` / `证据:`) — evidence-on-close. | +| `fable_evidence_log.py` | `PostToolUse` (Bash) | Passive recorder: appends every command's **real outcome** (command, exit code, output tail) to `.fable/evidence.jsonl` — the machine-written record citations are checked against. Records even while PAUSED. | +| `fable_close_guard.py` | `Stop` | While the ledger still has unchecked items, **block ending the turn** (cures early stopping / spinning). When all items are checked: **block if any `- [x]` lacks an evidence marker** (`-- evidence:` / `证据:`), **block if a cited evidence `command` has no successful run in the evidence log** (machine corroboration), and with `REPLAY: on` **block if a cited acceptance fails when re-run now**. | `fable_lint.py` is **not a hook** — a one-shot CLI (`python3 fable_lint.py `) for wrap-up or CI: SPEC exists and carries source tags ([measured]/[inferred]/[not-shown] @@ -53,6 +54,7 @@ defaults to the conservative tier. This is SessionStart-only info (there is no - [x] 2. done -- evidence: pytest 21/21 - [~] 3. not this round -- deferred: reason PAUSED: reason <- optional line anywhere: suspend enforcement +REPLAY: on <- optional: re-run cited acceptances at turn-end ROUTING: frugal <- optional: model-routing profile for this round TIER: throughput <- optional: concurrency tier for this round ``` @@ -141,7 +143,10 @@ use your actual absolute clone path if it differs: "command": "python3 ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills/fable-mode/hooks/fable_spawn_guard.py"}]}], "PostToolUse": [{"matcher": "Bash", "hooks": [{"type": "command", - "command": "python3 ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills/fable-mode/hooks/fable_fail_streak.py"}]}], + "command": "python3 ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills/fable-mode/hooks/fable_fail_streak.py"}]}, + {"matcher": "Bash", + "hooks": [{"type": "command", + "command": "python3 ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills/fable-mode/hooks/fable_evidence_log.py"}]}], "Stop": [{"hooks": [{"type": "command", "command": "python3 ${CLAUDE_CONFIG_DIR:-$HOME/.claude}/skills/fable-mode/hooks/fable_close_guard.py"}]}] } @@ -160,6 +165,7 @@ To disable entirely, remove the hooks block from settings.json. No third-party deps, just run: ```bash -python3 tests/test_guards.py # 13 cases: opt-in detection, ledger presence, small-spawn/fork exemptions, git-root boundary, loop-safety, fail-open -python3 tests/test_inject.py # 9 cases: per-model tier, env override, ledger context recovery, JSON envelope, fail-open +python3 tests/test_guards.py # opt-in detection, ledger presence, exemptions, git-root boundary, loop-safety, fail-open +python3 tests/test_inject.py # per-model tier, env override, ledger context recovery, JSON envelope, fail-open +python3 tests/test_evidence.py # evidence log recording, citation corroboration, REPLAY re-runs ``` diff --git a/hooks/_fable_common.py b/hooks/_fable_common.py index 342cf27..40517d2 100755 --- a/hooks/_fable_common.py +++ b/hooks/_fable_common.py @@ -125,6 +125,128 @@ def load_session_model(session_id): EVIDENCE_RE = re.compile(r"(evidence|verified|证据|凭证|验证)\s*[:：]", re.IGNORECASE) +# --- machine-written evidence log (.fable/evidence.jsonl) --- +# +# The Evidence Logger hook appends one JSON line per Bash command: +# {"ts": , "cmd": , "exit": , "tail": } +# The Close Guard checks cited `commands` on `- [x]` cards against this log, +# so "the acceptance actually ran" is machine truth, not a self-reported note. + +EVIDENCE_LOG = "evidence.jsonl" +EVIDENCE_LOG_MAX_BYTES = 512 * 1024 # rotate: keep the newest half beyond this +EVIDENCE_TAIL_CHARS = 200 + +_BACKTICK_RE = re.compile(r"`([^`]+)`") + + +def evidence_log_path(fable_dir): + return os.path.join(fable_dir, EVIDENCE_LOG) + + +def response_exit_code(tool_response): + """Best-effort exit code from a Bash tool_response; None when unknown.""" + r = tool_response + if isinstance(r, str): + m = re.search(r"[Ee]xit code[: ]+([0-9]+)", r) + return int(m.group(1)) if m else None + if not isinstance(r, dict): + return None + for key in ("exitCode", "exit_code", "code", "returncode"): + v = r.get(key) + if isinstance(v, int): + return v + for key in ("is_error", "isError"): + if r.get(key) is True: + return 1 + text = " ".join(str(r.get(k, "")) for k in ("stdout", "stderr", "output")) + m = re.search(r"[Ee]xit code[: ]+([0-9]+)", text) + return int(m.group(1)) if m else None + + +def append_evidence(fable_dir, cmd, exit_code, tail): + """Append one run record; rotate the log when it grows too large. + Best-effort, fail-open — recording must never disturb the session.""" + try: + path = evidence_log_path(fable_dir) + try: + if os.path.getsize(path) > EVIDENCE_LOG_MAX_BYTES: + with open(path, encoding="utf-8", errors="replace") as fh: + lines = fh.readlines() + with open(path, "w", encoding="utf-8") as fh: + fh.writelines(lines[len(lines) // 2:]) + except OSError: + pass + rec = {"ts": time.time(), "cmd": str(cmd)[:2000], + "exit": exit_code, + "tail": str(tail or "")[-EVIDENCE_TAIL_CHARS:]} + with open(path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + except Exception: + pass + + +def _norm_cmd(s): + return re.sub(r"\s+", " ", str(s)).strip() + + +def cited_commands(card_line): + """Backtick-quoted commands in the *evidence part* of a `- [x]` line. + Returns [] when the evidence note cites no command (prose-only note).""" + m = EVIDENCE_RE.search(card_line) + if not m: + return [] + return [_norm_cmd(c) for c in _BACKTICK_RE.findall(card_line[m.end():]) + if _norm_cmd(c)] + + +def evidence_log_has_run(log_path, cited, want_success=True): + """True if the log records a run whose command matches `cited` + (normalized substring, either direction) — successful when want_success.""" + try: + with open(log_path, encoding="utf-8", errors="replace") as fh: + for line in fh: + try: + rec = json.loads(line) + except ValueError: + continue + cmd = _norm_cmd(rec.get("cmd", "")) + if not cmd: + continue + if cited in cmd or cmd in cited: + if not want_success or rec.get("exit") == 0: + return True + except Exception: + return False + return False + + +def uncorroborated_citations(ledger_p, log_path): + """`- [x]` cards whose cited evidence command never ran successfully. + + Machine check for "the acceptance actually ran": a card that cites a + `command` as evidence must have a successful run of that command in the + evidence log. Cards with prose-only evidence are not checked here (the + substantive-string rule still applies to them). Returns [] when the log + doesn't exist yet (projects predating the logger) — fail-open. + """ + if not os.path.isfile(log_path): + return [] + bad = [] + try: + with open(ledger_p, encoding="utf-8", errors="replace") as fh: + for line in fh: + s = line.strip() + if s[:5].lower() != "- [x]": + continue + cites = cited_commands(s) + if cites and not any( + evidence_log_has_run(log_path, c) for c in cites): + bad.append(s) + except Exception: + return [] + return bad + + # --- model-routing profiles (quality / balanced / frugal) --- ROUTING_PROFILES = ("quality", "balanced", "frugal") @@ -151,6 +273,22 @@ def read_tier(path): return None +_REPLAY_RE = re.compile(r"^REPLAY\s*[:：]\s*(on|off)\b", re.IGNORECASE) + + +def read_replay(path): + """True when the ledger opts into acceptance replay (`REPLAY: on`).""" + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + m = _REPLAY_RE.match(line.strip()) + if m: + return m.group(1).lower() == "on" + except Exception: + return False + return False + + def read_routing(path): """Per-round routing profile from a `ROUTING: ` ledger line. diff --git a/hooks/fable_close_guard.py b/hooks/fable_close_guard.py index e0158a4..0a8bba9 100755 --- a/hooks/fable_close_guard.py +++ b/hooks/fable_close_guard.py @@ -22,10 +22,66 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _fable_common import ( # noqa: E402 read_hook_input, start_dir, find_fable_dir, ledger_path, parse_ledger, - closed_without_evidence, + closed_without_evidence, evidence_log_path, uncorroborated_citations, + read_replay, cited_commands, ) MAX_LIST = 12 +REPLAY_CMD_TIMEOUT = 30 # seconds per acceptance command (FABLE_REPLAY_TIMEOUT) +REPLAY_TOTAL_BUDGET = 120 # seconds across all replays in one stop + + +def replay_failures(ledger_p, project_root): + """Re-run each `- [x]` card's cited acceptance command; list the failures. + + Only runs when the ledger has `REPLAY: on` (checked by the caller) — an + explicit opt-in, because re-running acceptances at every stop costs real + time. A command that exits non-zero or times out is a failure: 'it passed + once' is not 'it still passes'. Budgeted so a heavy suite can't hang the + stop indefinitely. Fail-open on unexpected errors. + """ + import subprocess + try: + timeout = int(os.environ.get("FABLE_REPLAY_TIMEOUT", + str(REPLAY_CMD_TIMEOUT))) + except ValueError: + timeout = REPLAY_CMD_TIMEOUT + failures = [] + seen = set() + spent = 0.0 + try: + with open(ledger_p, encoding="utf-8", errors="replace") as fh: + lines = [l.strip() for l in fh] + except Exception: + return [] + import time as _time + for s in lines: + if s[:5].lower() != "- [x]": + continue + for cmd in cited_commands(s): + if cmd in seen: + continue + seen.add(cmd) + if spent >= REPLAY_TOTAL_BUDGET: + failures.append((cmd, "not replayed: %ds replay budget spent " + "(raise FABLE_REPLAY_TIMEOUT or drop " + "REPLAY: on)" % REPLAY_TOTAL_BUDGET)) + continue + t0 = _time.time() + try: + p = subprocess.run(cmd, shell=True, cwd=project_root, + capture_output=True, text=True, + timeout=min(timeout, + REPLAY_TOTAL_BUDGET - spent)) + if p.returncode != 0: + tail = (p.stderr or p.stdout or "").strip()[-160:] + failures.append((cmd, "exit %d: %s" % (p.returncode, tail))) + except subprocess.TimeoutExpired: + failures.append((cmd, "timed out")) + except Exception as e: + failures.append((cmd, "could not run: %r" % e)) + spent += _time.time() - t0 + return failures def main(): @@ -65,6 +121,45 @@ def main(): "evidence.\n" % (len(bad), path, lines) ) return 2 + # Machine corroboration: a card that cites a `command` as evidence + # must have a successful run of that command in the evidence log + # (written by the Evidence Logger hook, not by the model). + unc = uncorroborated_citations(path, evidence_log_path(fable_dir)) + if unc: + shown = unc[:MAX_LIST] + lines = "\n".join(" " + it for it in shown) + if len(unc) > len(shown): + lines += "\n ... and %d more" % (len(unc) - len(shown)) + sys.stderr.write( + "[fable-mode] BLOCKED stop: %d checked card(s) cite an " + "evidence `command` with NO successful run recorded in the " + "evidence log (%s):\n%s\n" + "The log is written by the Evidence Logger hook from real " + "tool results — a cited command that never ran (or never " + "exited 0) is not evidence. Run the acceptance command now, " + "or fix the citation to the command that actually ran.\n" + % (len(unc), evidence_log_path(fable_dir), lines) + ) + return 2 + # Acceptance replay (opt-in via `REPLAY: on`): 'passed once' is not + # 'still passes' — re-run each card's cited acceptance before the + # round may end, so a later card can't silently break an earlier one. + if read_replay(path): + fails = replay_failures(path, os.path.dirname(fable_dir)) + if fails: + shown = fails[:MAX_LIST] + lines = "\n".join(" `%s` -> %s" % f for f in shown) + if len(fails) > len(shown): + lines += "\n ... and %d more" % (len(fails) - len(shown)) + sys.stderr.write( + "[fable-mode] BLOCKED stop: REPLAY is on and %d cited " + "acceptance command(s) do not pass when re-run now:\n%s\n" + "A card whose acceptance no longer passes is not done — " + "fix the regression (or, if the command is genuinely " + "stale, fix the citation), then stop.\n" + % (len(fails), lines) + ) + return 2 return 0 # all closed, all evidenced -> allow stop shown = open_items[:MAX_LIST] diff --git a/hooks/fable_evidence_log.py b/hooks/fable_evidence_log.py new file mode 100644 index 0000000..3e0a17a --- /dev/null +++ b/hooks/fable_evidence_log.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""fable-mode Evidence Logger (PostToolUse hook on Bash). + +Machine-written evidence: appends every Bash command's real outcome +(command, exit code, output tail) to `.fable/evidence.jsonl`. The Close +Guard then verifies that any `command` a `- [x]` card cites as evidence +actually ran — and succeeded — in this recorded history. + +This closes the honor-system gap: a model can type `-- evidence: pytest +21/21` without ever running pytest, but it cannot forge an entry in a log +only this hook writes. (It could still edit the file by hand — the log is +tamper-evident-by-convention, not cryptographic — but a fabrication now +requires a visible, auditable act instead of a plausible sentence.) + +Passive recorder: always exit 0, never blocks, never prints. Armed per +project by `.fable/`; records even while PAUSED (pausing enforcement must +not create evidence gaps). Fail-open on any error. +""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from _fable_common import ( # noqa: E402 + read_hook_input, start_dir, find_fable_dir, append_evidence, + response_exit_code, +) + + +def main(): + data = read_hook_input() + fable_dir = find_fable_dir(start_dir(data)) + if not fable_dir: + return 0 # not opted in -> inert + + tool_input = data.get("tool_input") or {} + cmd = tool_input.get("command") if isinstance(tool_input, dict) else None + if not cmd: + return 0 + + r = data.get("tool_response") + exit_code = response_exit_code(r) + tail = "" + if isinstance(r, dict): + tail = str(r.get("stdout") or r.get("output") or r.get("stderr") or "") + elif isinstance(r, str): + tail = r + append_evidence(fable_dir, cmd, exit_code, tail) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as e: # passive recorder: never disturb the session + sys.stderr.write("[fable-mode] evidence log error (ignored): %r\n" % e) + sys.exit(0) diff --git a/hooks/fable_profile_inject.py b/hooks/fable_profile_inject.py index 7f1275a..dddfcfe 100644 --- a/hooks/fable_profile_inject.py +++ b/hooks/fable_profile_inject.py @@ -142,7 +142,11 @@ def build_context(profile, model, ledger_state, open_items, routing): "probes; tag SPEC decisions [measured]/[inferred]/[not-shown]. " "Guards block spawning without OPEN cards, stopping with open cards, " "and checking a card `- [x]` without a substantive `-- evidence:` " - "note.", + "note. Cite the acceptance `command` in the evidence note — cited " + "commands are corroborated against the machine-written evidence log " + "(.fable/evidence.jsonl); a citation that never ran (or never exited " + "0) blocks the stop. Optional ledger directive: `REPLAY: on` (cited " + "acceptances re-run before the round may end).", "Fable-5 habits: (1) audit every progress claim against a tool " "result — unverified means say 'unverified'; (2) don't end the turn " "on an actionable plan/promise — act now; (3) lead with the outcome; " diff --git a/templates/LEDGER.template.md b/templates/LEDGER.template.md index 02bc3e9..4a6534a 100644 --- a/templates/LEDGER.template.md +++ b/templates/LEDGER.template.md @@ -11,6 +11,12 @@ Only mark `- [x]` after the acceptance command actually ran — the Close Guard blocks turn-end for any `- [x]` without an evidence note (`-- evidence:` / `verified:` / `证据:`): +Cite the acceptance `command` inside the evidence note — the Close Guard +corroborates cited commands against `.fable/evidence.jsonl` (machine-written +by the Evidence Logger): a citation that never ran, or never exited 0, blocks +the stop. Optional: a `REPLAY: on` line re-runs cited acceptances before the +round may end ('passed once' is not 'still passes'). + - [ ] 1. — acceptance: `` - [ ] 2. — acceptance: `` - [x] 0. (example) scaffold -- evidence: `pytest -q` -> 12 passed diff --git a/tests/test_evidence.py b/tests/test_evidence.py new file mode 100644 index 0000000..a38329c --- /dev/null +++ b/tests/test_evidence.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Tests for machine-written evidence: the Evidence Logger hook, the Close +Guard's citation corroboration, and REPLAY re-runs. Same conventions as +test_guards.py.""" +import json, os, subprocess, tempfile, shutil, sys + +HOOKS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "hooks") +CLOSE = os.path.join(HOOKS, "fable_close_guard.py") +EVLOG = os.path.join(HOOKS, "fable_evidence_log.py") + +passed = failed = 0 +def check(name, got, want): + global passed, failed + ok = got == want + print(("PASS" if ok else "FAIL"), name, "got=%s want=%s" % (got, want)) + if ok: passed += 1 + else: failed += 1 + +def run(script, payload): + p = subprocess.run([sys.executable, script], input=json.dumps(payload), + capture_output=True, text=True) + return p.returncode + +tmps = [] +def proj(ledger=None, evlog=None): + d = tempfile.mkdtemp(prefix="fbev_") + tmps.append(d) + os.mkdir(os.path.join(d, ".git")) + fd = os.path.join(d, ".fable"); os.mkdir(fd) + if ledger is not None: + with open(os.path.join(fd, "LEDGER.md"), "w") as f: f.write(ledger) + if evlog is not None: + with open(os.path.join(fd, "evidence.jsonl"), "w") as f: + for rec in evlog: + f.write(json.dumps(rec) + "\n") + return d + +OK = {"ts": 1.0, "exit": 0} + +# ---- Evidence Logger (recorder) ---- +# 1. appends a record with the real command and exit code +d = proj(ledger="- [ ] 1. card\n") +run(EVLOG, {"cwd": d, "tool_name": "Bash", + "tool_input": {"command": "pytest -q"}, + "tool_response": {"stdout": "21 passed", "exitCode": 0}}) +lp = os.path.join(d, ".fable", "evidence.jsonl") +rec = json.loads(open(lp).read().strip()) +check("evlog/records-cmd", rec["cmd"], "pytest -q") +check("evlog/records-exit", rec["exit"], 0) + +# 2. not opted in -> writes nothing +d2 = tempfile.mkdtemp(prefix="fbev_"); tmps.append(d2) +os.mkdir(os.path.join(d2, ".git")) +run(EVLOG, {"cwd": d2, "tool_name": "Bash", + "tool_input": {"command": "echo x"}, + "tool_response": {"exitCode": 0}}) +check("evlog/inert-without-optin", + os.path.exists(os.path.join(d2, ".fable", "evidence.jsonl")), False) + +# 3. records even while PAUSED (evidence gaps are worse than pauses) +d = proj(ledger="- [ ] 1. card\nPAUSED: side work\n") +run(EVLOG, {"cwd": d, "tool_name": "Bash", + "tool_input": {"command": "echo paused"}, + "tool_response": {"exitCode": 0}}) +check("evlog/records-while-paused", + os.path.exists(os.path.join(d, ".fable", "evidence.jsonl")), True) + +# ---- Close Guard: citation corroboration ---- +# 1. cited command with successful run in log -> allow +d = proj(ledger="- [x] 1. done -- evidence: `pytest -q` 21 passed\n", + evlog=[dict(OK, cmd="cd /x && pytest -q", tail="21 passed")]) +check("corroborate/cited-and-ran-allows", run(CLOSE, {"cwd": d}), 0) + +# 2. cited command NOT in log -> BLOCK (fabricated citation) +d = proj(ledger="- [x] 1. done -- evidence: `pytest -q` 21 passed\n", + evlog=[dict(OK, cmd="echo hello", tail="hello")]) +check("corroborate/cited-never-ran-blocks", run(CLOSE, {"cwd": d}), 2) + +# 3. cited command ran but FAILED -> BLOCK +d = proj(ledger="- [x] 1. done -- evidence: `pytest -q` all good\n", + evlog=[{"ts": 1.0, "cmd": "pytest -q", "exit": 1, "tail": "2 failed"}]) +check("corroborate/cited-but-failed-blocks", run(CLOSE, {"cwd": d}), 2) + +# 4. no log file at all (pre-logger project) -> old behavior, allow +d = proj(ledger="- [x] 1. done -- evidence: `pytest -q` 21 passed\n") +check("corroborate/no-log-failopen", run(CLOSE, {"cwd": d}), 0) + +# 5. prose-only evidence (no backtick command) -> not machine-checked, allow +d = proj(ledger="- [x] 1. done -- evidence: screenshot at docs/x.png\n", + evlog=[dict(OK, cmd="echo hi", tail="hi")]) +check("corroborate/prose-evidence-unchecked", run(CLOSE, {"cwd": d}), 0) + +# 6. acceptance backtick BEFORE the evidence marker is not a citation +d = proj(ledger="- [x] 1. thing — acceptance: `make test` -- evidence: " + "ran the suite, 40 green\n", + evlog=[dict(OK, cmd="echo unrelated", tail="")]) +check("corroborate/acceptance-part-not-cited", run(CLOSE, {"cwd": d}), 0) + +# 7. PAUSED still disables the close guard entirely (regression) +d = proj(ledger="- [x] 1. done -- evidence: `pytest -q` ok\nPAUSED: side\n", + evlog=[dict(OK, cmd="echo other", tail="")]) +check("corroborate/paused-allows", run(CLOSE, {"cwd": d}), 0) + +# ---- Close Guard: REPLAY re-runs ---- +# 1. replay armed, cited acceptance passes when re-run -> allow +d = proj(ledger="REPLAY: on\n- [x] 1. done -- evidence: `true` clean exit\n", + evlog=[dict(OK, cmd="true", tail="")]) +check("replay/pass-allows", run(CLOSE, {"cwd": d}), 0) + +# 2. replay armed, cited command now fails -> BLOCK (regression caught) +d = proj(ledger="REPLAY: on\n- [x] 1. done -- evidence: `false` ok\n", + evlog=[dict(OK, cmd="false", tail="")]) +check("replay/fail-blocks", run(CLOSE, {"cwd": d}), 2) + +# 3. no REPLAY line -> no replay (the same failing card passes the stop) +d = proj(ledger="- [x] 1. done -- evidence: `false` ok\n", + evlog=[dict(OK, cmd="false", tail="")]) +check("replay/off-by-default", run(CLOSE, {"cwd": d}), 0) + +for d in tmps: + shutil.rmtree(d, ignore_errors=True) + +print("\n%d passed, %d failed" % (passed, failed)) +sys.exit(1 if failed else 0) diff --git a/tests/test_install.py b/tests/test_install.py index 534250d..bcf7639 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -6,7 +6,10 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) INSTALL_SRC = os.path.join(REPO, "install.sh") NAMES = ["fable_profile_inject.py", "fable_spawn_guard.py", - "fable_fail_streak.py", "fable_close_guard.py"] + "fable_fail_streak.py", "fable_evidence_log.py", + "fable_close_guard.py"] +# expected hook-group count per event (PostToolUse carries two: streak + evidence log) +EXPECT = {"SessionStart": 1, "PreToolUse": 1, "PostToolUse": 2, "Stop": 1} passed = failed = 0 @@ -62,7 +65,8 @@ def cmds(d): # B. idempotent — second run must not duplicate run(skill, cfg) d = load(cfg) - check("install/idempotent", all(len(d["hooks"][e]) == 1 for e in d["hooks"])) + check("install/idempotent", + all(len(d["hooks"][e]) == EXPECT[e] for e in d["hooks"])) # C. merge — preserve unrelated config + a user's own hook with open(os.path.join(cfg, "settings.json"), "w") as f: From 9a67155c2d75e26c50f11d758f15e4a47aeef34f Mon Sep 17 00:00:00 2001 From: Eliauk Date: Thu, 16 Jul 2026 09:42:18 +0200 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20discipline=20hardening=20=E2=80=94?= =?UTF-8?q?=20verifier=20isolation,=20MODE:=20light=20triage,=20structural?= =?UTF-8?q?=20fail-streak?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three quality mechanisms, one theme: make the existing levers bind where they currently rely on willpower. 1. Verifier information isolation (VERIFIER_PROMPT.md + SKILL.md): a verifier that reads the worker's transcript, summary, or claimed status grades the *story*, not the work — contaminated context is why self-critique underperforms. Hard rules for the dispatcher: verifier gets ONLY the SPEC excerpt + artifact; no expected verdict; 2-3 *different-lens* verifiers over N identical ones; never weaker than the implementer. 2. MODE: light (ledger directive): triage for small rounds. Full ceremony (SPEC + cards + evidence + replay) earns its cost on long, multi-file, hard-to-reverse work; on a small immediately-verifiable round it is overhead the model learns to game. Light keeps the honesty rules (evidence-on-close, corroboration, ceiling, fail-streak) and drops the ceremony (design gate, open-cards-block-stop). Choosing the weight is part of the discipline. 3. Structural fail-streak: the attribution-ladder reminder stays advisory at 3, but at 6 consecutive failures insight has demonstrably not worked — every further failure now exits 2 with a demand to stop retrying, distill '-- tried: ' into the card, and restart from a fresh context. The note (or a success) resets the guard: the distillation is the exit, so the lesson survives the context that learned it. Grinding in a failure-polluted context makes models dumber; this makes the documented restart-fresh rule mechanical instead of aspirational. Invariants preserved: fail-open, .fable/ opt-in, loop-safe, stdlib only. Tests: tests/test_discipline.py (13 cases); all existing suites green. Co-Authored-By: Claude Fable 5 --- SKILL.md | 9 ++- hooks/README.md | 7 +- hooks/_fable_common.py | 19 +++++ hooks/fable_close_guard.py | 4 +- hooks/fable_fail_streak.py | 126 +++++++++++++++++++++++----------- hooks/fable_profile_inject.py | 7 +- hooks/fable_spawn_guard.py | 5 +- templates/LEDGER.template.md | 8 ++- templates/VERIFIER_PROMPT.md | 22 +++++- tests/test_discipline.py | 115 +++++++++++++++++++++++++++++++ 10 files changed, 271 insertions(+), 51 deletions(-) create mode 100644 tests/test_discipline.py diff --git a/SKILL.md b/SKILL.md index 44436e4..4d82cca 100644 --- a/SKILL.md +++ b/SKILL.md @@ -47,7 +47,7 @@ Before code, write `docs/SPEC.md`: requirements, approach, **task cards** (skele Each card runs in a **fresh context**, fed only the relevant SPEC excerpt — no reasoning garbage from prior cards. Run acceptance the moment it's done; **don't advance until it passes**. Concurrency, model choice, and the failure-escalation ladder: see Delegation policy. ### 3. Adversarial self-check -Important output is never "generate and ship". Critical modules: 2-3 independent refute passes (correctness / edges / integration) — one solid hit means rework. Wide solution spaces: N approaches + judge + synthesize. **Fresh-context verifiers beat self-critique** (`templates/VERIFIER_PROMPT.md`); verifier prompts say "assume broken, falsify hard", never "take a look". +Important output is never "generate and ship". Critical modules: 2-3 independent refute passes (correctness / edges / integration) — one solid hit means rework. Wide solution spaces: N approaches + judge + synthesize. **Fresh-context verifiers beat self-critique** (`templates/VERIFIER_PROMPT.md`); verifier prompts say "assume broken, falsify hard", never "take a look". **Information isolation is what makes it adversarial**: the verifier gets ONLY the SPEC excerpt + the artifact — never the worker's transcript, notes, claimed evidence, or expected verdict; a verifier that reads the worker's story grades the story, not the work. Prefer 2-3 verifiers with *different* lenses over N identical ones, and never a verifier weaker than the implementer. **Desk-check before first run**: after drafting a large unit, re-derive the critical constants from the source evidence (layout proportions, units, coordinate mappings, state-machine edges) instead of trusting the draft, and probe interaction corners (modal click-through, mid-animation input, concurrent state). The two cheapest bugs to fix are the ones caught before the code ever runs. @@ -136,6 +136,7 @@ Five hooks turn the most-shirked rules into hard blocks. Armed **per project** b - [ ] 1. card (machine-checkable acceptance) <- open: guards enforce - [x] 2. done -- evidence: `pytest -q` 21/21 <- [x] REQUIRES substantive evidence; a cited `command` is checked against the machine-written evidence log - [~] 3. not this round -- deferred: reason +MODE: light <- optional: light round (triage) — ceremony off, honesty rules stay REPLAY: on <- optional: re-run cited acceptances before the round may end PAUSED: reason <- a line anywhere: enforcement off ``` @@ -145,14 +146,16 @@ be attributable. Evidence notes must be substantive: `evidence: ok` counts as missing.) - **Spawn Guard** (PreToolUse Agent/Task/Workflow): blocks a detailed spawn while the ledger has no **open** cards — no ledger, and equally a ledger holding only a finished round's closed cards (design gate: new fan-out needs a live card) — and blocks any spawn requesting a **model stronger than the session's** (model ceiling — checked on the `model` param and `model:` literals in Workflow scripts; stays active even when paused, it protects quota, not workflow). -- **Fail-Streak Reminder** (PostToolUse Bash, advisory): every 3rd consecutive failing command injects the attribution ladder — stops grinding on the wrong layer mechanically, not by willpower. +- **Fail-Streak Guard** (PostToolUse Bash): every 3rd consecutive failing command injects the attribution ladder (advisory); at the **6th** it turns structural — every further failure is answered with a blocking demand to stop retrying, distill `-- tried: ` into the card, and restart from a fresh context. Writing the note (or a success) resets it: the distillation is the exit, so the lesson survives the context that learned it. - **Evidence Logger** (PostToolUse Bash, passive): appends every command's real outcome (command, exit code, output tail) to `.fable/evidence.jsonl` — the machine-written record the Close Guard checks citations against. Records even while paused; evidence gaps are worse than pauses. - **Close Guard** (Stop): blocks ending the turn while open `- [ ]` items remain, while any `- [x]` lacks an `-- evidence:` note (evidence-on-close: adjectives don't close cards), while any cited evidence `command` has **no successful run in the evidence log** (machine corroboration: a citation that never ran is not evidence), and — with `REPLAY: on` — while any cited acceptance fails when **re-run now** ('passed once' is not 'still passes'). - **Profile Injector** (SessionStart): injects tier + routing + habits, **sized to the ledger state** — full when a round is starting/active, minimal when idle, one line when paused. **Wrap-up lint**: `python3 /hooks/fable_lint.py ` — machine-checks the discipline itself (SPEC source tags present, open cards name acceptance, closed cards carry evidence). Run it at step 7 of the execution template; findings are open work. -**Per-task granularity**: *active* (open cards) = full enforcement; *idle* (no/all-closed cards) = close guard quiet and small tasks flow freely, but a **detailed** fan-out still needs a live card first; *paused* (a `PAUSED: reason` line) = guards off except the ceiling. Write PAUSED only when the user steers to work unrelated to the round; remove it to resume. Small spawns (<1500 chars) and forks skip the design gate; everything fails open (a guard bug never bricks the session); loop-safe. +**Per-task granularity**: *active* (open cards) = full enforcement; *idle* (no/all-closed cards) = close guard quiet and small tasks flow freely, but a **detailed** fan-out still needs a live card first; *light* (a `MODE: light` line) = triage for small rounds — design gate and open-cards-block-stop off, evidence honesty and the ceiling stay armed; *paused* (a `PAUSED: reason` line) = guards off except the ceiling. Write PAUSED only when the user steers to work unrelated to the round; remove it to resume. Small spawns (<1500 chars) and forks skip the design gate; everything fails open (a guard bug never bricks the session); loop-safe. + +**Triage — pick the round's weight deliberately**: full ceremony (SPEC + cards + evidence + replay) earns its cost on long, multi-file, hard-to-reverse work; on a small, immediately-verifiable round it is overhead the model will learn to game. Start a small round with `MODE: light`; upgrade to full the moment scope grows past a couple of files or the work becomes hard to verify by eye. Choosing the weight is part of the discipline — applying maximum ceremony everywhere is not rigor, it's noise. **For substantial work: after writing the SPEC, `mkdir .fable` + create `.fable/LEDGER.md` to get the mechanical backstop.** In the user's repo, suggest gitignoring `.fable/` (round state) while committing `docs/SPEC.md`/`PROGRESS.md` (durable docs). diff --git a/hooks/README.md b/hooks/README.md index 8cb113d..a01fe6f 100644 --- a/hooks/README.md +++ b/hooks/README.md @@ -10,7 +10,7 @@ built around this repo's SPEC.md/PROGRESS.md conventions. |---|---|---| | `fable_profile_inject.py` | `SessionStart` | When the project has opted in, **auto-inject the tier by model + the six levers + ledger context recovery** (no need to type "use fable mode") | | `fable_spawn_guard.py` | `PreToolUse` (Agent\|Task\|Workflow) | When opted in: **block a detailed spawn with no ledger** (forces the plan gate) and **block any spawn requesting a model stronger than the session's** (the model ceiling) | -| `fable_fail_streak.py` | `PostToolUse` (Bash) | Advisory, never blocks: at every 3rd **consecutive failing command**, inject the attribution ladder (harness → deployment → product; fix the class via an invariant). Streak state: `$TMPDIR/fable-mode-sessions/.fails`, reset on success. | +| `fable_fail_streak.py` | `PostToolUse` (Bash) | At every 3rd **consecutive failing command**, inject the attribution ladder (advisory). At the **6th**, turn structural: every further failure exits 2 with a demand to stop retrying, write `-- tried: ` into the card, and restart fresh — the note (or a success) resets the streak. State: `$TMPDIR/fable-mode-sessions/.fails`. | | `fable_evidence_log.py` | `PostToolUse` (Bash) | Passive recorder: appends every command's **real outcome** (command, exit code, output tail) to `.fable/evidence.jsonl` — the machine-written record citations are checked against. Records even while PAUSED. | | `fable_close_guard.py` | `Stop` | While the ledger still has unchecked items, **block ending the turn** (cures early stopping / spinning). When all items are checked: **block if any `- [x]` lacks an evidence marker** (`-- evidence:` / `证据:`), **block if a cited evidence `command` has no successful run in the evidence log** (machine corroboration), and with `REPLAY: on` **block if a cited acceptance fails when re-run now**. | @@ -54,6 +54,7 @@ defaults to the conservative tier. This is SessionStart-only info (there is no - [x] 2. done -- evidence: pytest 21/21 - [~] 3. not this round -- deferred: reason PAUSED: reason <- optional line anywhere: suspend enforcement +MODE: light <- optional: light round — ceremony off, honesty stays REPLAY: on <- optional: re-run cited acceptances at turn-end ROUTING: frugal <- optional: model-routing profile for this round TIER: throughput <- optional: concurrency tier for this round @@ -93,7 +94,8 @@ state, so small tasks in a big project aren't taxed: | starting (no cards yet) | design gate armed | full (~1.6KB) | | **active** (open `- [ ]`) | full enforcement | full + context recovery | | **idle** (all closed) | close guard quiet; detailed fan-out still needs a new open card | one-liner (~0.4KB) | -| **paused** (`PAUSED: reason` line) | off except model ceiling | one-liner (~0.2KB) | +| **light** (`MODE: light` line) | design gate + open-cards-block-stop off; evidence honesty, evidence log, fail-streak and ceiling stay | full | +| **paused** (`PAUSED: reason` line) | off except model ceiling + evidence log | one-liner (~0.2KB) | ## Model ceiling (mechanical) @@ -168,4 +170,5 @@ No third-party deps, just run: python3 tests/test_guards.py # opt-in detection, ledger presence, exemptions, git-root boundary, loop-safety, fail-open python3 tests/test_inject.py # per-model tier, env override, ledger context recovery, JSON envelope, fail-open python3 tests/test_evidence.py # evidence log recording, citation corroboration, REPLAY re-runs +python3 tests/test_discipline.py # MODE: light triage, structural fail-streak ``` diff --git a/hooks/_fable_common.py b/hooks/_fable_common.py index 40517d2..8373f3d 100755 --- a/hooks/_fable_common.py +++ b/hooks/_fable_common.py @@ -274,6 +274,25 @@ def read_tier(path): _REPLAY_RE = re.compile(r"^REPLAY\s*[:：]\s*(on|off)\b", re.IGNORECASE) +_MODE_RE = re.compile(r"^MODE\s*[:：]\s*(light|full)\b", re.IGNORECASE) + + +def read_mode(path): + """Per-round ceremony weight from a `MODE: light|full` ledger line. + + 'light' = triage for small rounds: the design gate and open-cards-block- + stop are off, but evidence honesty (and the model ceiling) stay armed. + Default 'full'. Fail-open to 'full' on any read problem. + """ + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + m = _MODE_RE.match(line.strip()) + if m: + return m.group(1).lower() + except Exception: + return "full" + return "full" def read_replay(path): diff --git a/hooks/fable_close_guard.py b/hooks/fable_close_guard.py index 0a8bba9..ed78d49 100755 --- a/hooks/fable_close_guard.py +++ b/hooks/fable_close_guard.py @@ -23,7 +23,7 @@ from _fable_common import ( # noqa: E402 read_hook_input, start_dir, find_fable_dir, ledger_path, parse_ledger, closed_without_evidence, evidence_log_path, uncorroborated_citations, - read_replay, cited_commands, + read_mode, read_replay, cited_commands, ) MAX_LIST = 12 @@ -103,6 +103,8 @@ def main(): open_items, _has_any, paused = parse_ledger(path) if paused: return 0 # round paused -> enforcement off + if read_mode(path) == "light": + open_items = [] # light round: open cards don't block the stop if not open_items: # All cards closed -> enforce evidence-on-close before allowing stop. bad = closed_without_evidence(path) diff --git a/hooks/fable_fail_streak.py b/hooks/fable_fail_streak.py index fda0fb9..c616738 100644 --- a/hooks/fable_fail_streak.py +++ b/hooks/fable_fail_streak.py @@ -1,20 +1,30 @@ #!/usr/bin/env python3 -"""fable-mode Fail-Streak Reminder (PostToolUse hook on Bash). +"""fable-mode Fail-Streak Guard (PostToolUse hook on Bash). Grinding is the failure mode this catches: N consecutive failing commands -usually means the model is patching the wrong layer. At every 3rd consecutive -Bash failure it injects the attribution ladder as context: - - harness -> deployment -> product - -(1) suspect the test/driver itself, (2) prove the new code is actually -running (cache/build/restart), (3) only then debug the product — and fix the -class via an invariant, not the instance. - -Advisory only — never blocks (exit 0 always). Armed per project by `.fable/`; -off while the ledger is PAUSED. Streak state lives beside the model cache in -$TMPDIR/fable-mode-sessions/.fails and self-resets on the next success. -Fail-open on any error. +usually means the model is patching the wrong layer — and every further +attempt happens in a context polluted by the failures before it. + +Two rungs: + + streak 3 (advisory, every 3rd): inject the attribution ladder — + harness -> deployment -> product + (1) suspect the test/driver itself, (2) prove the new code is actually + running (cache/build/restart), (3) only then debug the product — and fix + the class via an invariant, not the instance. + + streak >= 6 (structural, exit 2): insight alone hasn't worked — now the + reset is mechanical. Every further failing command is answered with a + blocking demand: STOP retrying; distill what was ruled out into the + ledger card as `-- tried: `; then + restart the card from a FRESH context (subagent or new session) that + reads only SPEC + LEDGER — not the failure pile. Writing the `-- tried:` + note (or a success) is what resets the streak: the distillation is the + exit, so the lesson survives the context that learned it. + +Armed per project by `.fable/`; off while the ledger is PAUSED. Streak +state lives beside the model cache in $TMPDIR/fable-mode-sessions/ +.fails and self-resets on the next success. Fail-open on any error. """ import json import os @@ -24,12 +34,11 @@ class via an invariant, not the instance. sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _fable_common import ( # noqa: E402 read_hook_input, start_dir, find_fable_dir, ledger_path, parse_ledger, - load_fail_streak, save_fail_streak, + load_fail_streak, save_fail_streak, response_exit_code, ) REMIND_EVERY = 3 - -_EXIT_CODE_RE = re.compile(r"[Ee]xit code[: ]+([0-9]+)") +HARD_AT = 6 LADDER = ( "[fable-mode] %d consecutive failing commands — before the next fix, walk " @@ -42,25 +51,51 @@ class via an invariant, not the instance. "command keeps failing verbatim, stop retrying it." ) +RESET_DEMAND = ( + "[fable-mode] %d consecutive failing commands — this is a grind, and " + "further attempts inside this failure pile get WORSE, not better. Do not " + "run another fix attempt. Instead: (1) append to the current ledger card " + "a distillation `-- tried: `; (2) restart the card from a FRESH context — a " + "subagent or new session that reads only SPEC + LEDGER, not this " + "transcript. Writing the `-- tried:` note (or a passing command) resets " + "this guard; more grinding does not." +) + +# a `tried:` distillation line in the ledger is the structured exit +_TRIED_RE = re.compile(r"(tried|已试|排除)\s*[:：]", re.IGNORECASE) + + +def count_tried(lp): + try: + with open(lp, encoding="utf-8", errors="replace") as fh: + return sum(1 for line in fh if _TRIED_RE.search(line)) + except Exception: + return 0 -def command_failed(tool_response): - """Best-effort failure detection; uncertain -> treated as success.""" - r = tool_response - if isinstance(r, str): - return bool(_EXIT_CODE_RE.search(r) and - _EXIT_CODE_RE.search(r).group(1) != "0") - if not isinstance(r, dict): - return False - for key in ("exitCode", "exit_code", "code", "returncode"): - v = r.get(key) - if isinstance(v, int): - return v != 0 - for key in ("is_error", "isError"): - if r.get(key) is True: - return True - text = " ".join(str(r.get(k, "")) for k in ("stdout", "stderr", "output")) - m = _EXIT_CODE_RE.search(text) - return bool(m and m.group(1) != "0") + +def _tried_file(sid): + import tempfile + d = os.path.join(tempfile.gettempdir(), "fable-mode-sessions") + safe = re.sub(r"[^A-Za-z0-9._-]", "_", str(sid))[:120] + return os.path.join(d, safe + ".tried") + + +def load_tried(sid): + try: + with open(_tried_file(sid), encoding="utf-8") as fh: + return max(0, int(fh.read().strip() or 0)) + except Exception: + return 0 + + +def save_tried(sid, n): + try: + os.makedirs(os.path.dirname(_tried_file(sid)), exist_ok=True) + with open(_tried_file(sid), "w", encoding="utf-8") as fh: + fh.write(str(int(n))) + except Exception: + pass def main(): @@ -72,17 +107,30 @@ def main(): fable_dir = find_fable_dir(start_dir(data)) if not fable_dir: return 0 # not opted in -> inert - _open, _has, paused = parse_ledger(ledger_path(fable_dir)) + lp = ledger_path(fable_dir) + _open, _has, paused = parse_ledger(lp) if paused: return 0 - if not command_failed(data.get("tool_response")): + if response_exit_code(data.get("tool_response")) in (None, 0): if load_fail_streak(sid): save_fail_streak(sid, 0) + save_tried(sid, count_tried(lp)) return 0 - streak = load_fail_streak(sid) + 1 + # A new `-- tried:` distillation since the last failure is the structured + # exit from a grind: accept it and start fresh. + tried_now = count_tried(lp) + streak = load_fail_streak(sid) + if tried_now > load_tried(sid): + streak = 0 + streak += 1 save_fail_streak(sid, streak) + save_tried(sid, tried_now) + + if streak >= HARD_AT: + sys.stderr.write(RESET_DEMAND % streak + "\n") + return 2 # strongest PostToolUse signal: stderr fed back to Claude if streak >= REMIND_EVERY and streak % REMIND_EVERY == 0: print(json.dumps({ "hookSpecificOutput": { @@ -96,6 +144,6 @@ def main(): if __name__ == "__main__": try: sys.exit(main()) - except Exception as e: # advisory hook: never disturb the session + except Exception as e: # fail-open: never disturb the session sys.stderr.write("[fable-mode] fail-streak error (ignored): %r\n" % e) sys.exit(0) diff --git a/hooks/fable_profile_inject.py b/hooks/fable_profile_inject.py index dddfcfe..ce7f9c8 100644 --- a/hooks/fable_profile_inject.py +++ b/hooks/fable_profile_inject.py @@ -145,8 +145,11 @@ def build_context(profile, model, ledger_state, open_items, routing): "note. Cite the acceptance `command` in the evidence note — cited " "commands are corroborated against the machine-written evidence log " "(.fable/evidence.jsonl); a citation that never ran (or never exited " - "0) blocks the stop. Optional ledger directive: `REPLAY: on` (cited " - "acceptances re-run before the round may end).", + "0) blocks the stop. Optional ledger directives: `REPLAY: on` (cited " + "acceptances re-run before the round may end); `MODE: light` (small " + "round — design gate and open-cards-block-stop off, evidence honesty " + "and the ceiling stay). Verifiers get ONLY the SPEC excerpt + the " + "artifact — never your claims, transcript, or expected verdict.", "Fable-5 habits: (1) audit every progress claim against a tool " "result — unverified means say 'unverified'; (2) don't end the turn " "on an actionable plan/promise — act now; (3) lead with the outcome; " diff --git a/hooks/fable_spawn_guard.py b/hooks/fable_spawn_guard.py index 86d0beb..25c8aaf 100755 --- a/hooks/fable_spawn_guard.py +++ b/hooks/fable_spawn_guard.py @@ -28,7 +28,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _fable_common import ( # noqa: E402 read_hook_input, start_dir, find_fable_dir, ledger_path, parse_ledger, - model_tier, load_session_model, + model_tier, load_session_model, read_mode, ) @@ -108,6 +108,9 @@ def main(): if is_fork(tool_input): return 0 # forks inherit full context; exempt from the spec tax + if read_mode(ledger_path(fable_dir)) == "light": + return 0 # light round: ceremony off, ceiling stayed active above + try: threshold = int(os.environ.get("FABLE_SPAWN_MIN_CHARS", "1500")) except ValueError: diff --git a/templates/LEDGER.template.md b/templates/LEDGER.template.md index 4a6534a..b6a084f 100644 --- a/templates/LEDGER.template.md +++ b/templates/LEDGER.template.md @@ -14,8 +14,12 @@ blocks turn-end for any `- [x]` without an evidence note (`-- evidence:` / Cite the acceptance `command` inside the evidence note — the Close Guard corroborates cited commands against `.fable/evidence.jsonl` (machine-written by the Evidence Logger): a citation that never ran, or never exited 0, blocks -the stop. Optional: a `REPLAY: on` line re-runs cited acceptances before the -round may end ('passed once' is not 'still passes'). +the stop. Optional directive lines (one per round, auditable): +`MODE: light` (small round: ceremony off, evidence honesty stays) · +`REPLAY: on` (re-run cited acceptances before the round may end). +On a grind (6+ consecutive failures) the Fail-Streak Guard demands a +`-- tried: ` note on the card — writing it resets the +guard and is the handoff for a fresh-context restart. - [ ] 1. — acceptance: `` - [ ] 2. — acceptance: `` diff --git a/templates/VERIFIER_PROMPT.md b/templates/VERIFIER_PROMPT.md index abbaacf..2ad98ca 100644 --- a/templates/VERIFIER_PROMPT.md +++ b/templates/VERIFIER_PROMPT.md @@ -4,10 +4,30 @@ Use for lever 3 / habit 6: paste into a fresh context (subagent, second CLI session, or any other AI engine). Fill the <>. The generator must not be the only judge. +## Information isolation (hard rules for the DISPATCHER) + +Adversarial verification only works when the verifier cannot inherit the +worker's beliefs. A verifier that reads the worker's transcript, summary, or +self-assessment is contaminated — it will grade the *story*, not the work. + +1. The verifier receives ONLY: the SPEC excerpt (the card + its acceptance + criteria) and the artifact (diff / files / output). NEVER the worker's + transcript, progress notes, explanations, or claimed evidence. +2. Do not tell the verifier what you believe the status is ("it should pass + now", "I already fixed X") — no expected verdict, no framing. +3. The verifier must be at least as strong as the implementer. +4. For critical cards, run 2–3 verifiers with DIFFERENT lenses (correctness / + edges / integration below) rather than N identical ones — diverse lenses + catch failure modes redundancy can't. Treat "any verifier fails" as fail. +5. The verifier's verdict goes into the card's `-- evidence:` note with what + it actually ran — not adjectives. + --- You are a skeptical reviewer with no attachment to this work. Assume it is -broken; your job is to falsify it, not to approve it. +broken; your job is to falsify it, not to approve it. You are deliberately +given no account of how this work was produced or how confident its author +is — judge only what is in front of you. Specification (excerpt): diff --git a/tests/test_discipline.py b/tests/test_discipline.py new file mode 100644 index 0000000..6240a72 --- /dev/null +++ b/tests/test_discipline.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Tests for MODE: light triage and the structural fail-streak. Same +conventions as test_guards.py.""" +import json, os, subprocess, tempfile, shutil, sys, time + +HOOKS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "hooks") +SPAWN = os.path.join(HOOKS, "fable_spawn_guard.py") +CLOSE = os.path.join(HOOKS, "fable_close_guard.py") +STREAK = os.path.join(HOOKS, "fable_fail_streak.py") +INJ = os.path.join(HOOKS, "fable_profile_inject.py") + +passed = failed = 0 +def check(name, got, want): + global passed, failed + ok = got == want + print(("PASS" if ok else "FAIL"), name, "got=%s want=%s" % (got, want)) + if ok: passed += 1 + else: failed += 1 + +def run(script, payload): + p = subprocess.run([sys.executable, script], input=json.dumps(payload), + capture_output=True, text=True) + return p.returncode + +def run_out(script, payload): + p = subprocess.run([sys.executable, script], input=json.dumps(payload), + capture_output=True, text=True) + return p.returncode, p.stdout, p.stderr + +tmps = [] +def proj(ledger=None): + d = tempfile.mkdtemp(prefix="fbdisc_") + tmps.append(d) + os.mkdir(os.path.join(d, ".git")) + fd = os.path.join(d, ".fable"); os.mkdir(fd) + if ledger is not None: + with open(os.path.join(fd, "LEDGER.md"), "w") as f: f.write(ledger) + return d + +BIG = "x" * 2000 +SMALL = "y" * 100 +RUN_TAG = "%d" % (time.time() * 1000) + +# ---- MODE: light (triage) ---- +# 1. light mode: open cards do not block the stop +d = proj(ledger="MODE: light\n- [ ] 1. open card\n") +check("light/open-cards-dont-block", run(CLOSE, {"cwd": d}), 0) + +# 2. light mode: evidence-on-close still enforced +d = proj(ledger="MODE: light\n- [x] 1. done\n") +check("light/evidence-still-enforced", run(CLOSE, {"cwd": d}), 2) + +# 3. light mode: design gate off (big spawn without open cards allowed) +d = proj(ledger="MODE: light\n") +check("light/design-gate-off", run(SPAWN, {"cwd": d, "tool_name": "Agent", + "tool_input": {"prompt": BIG}}), 0) + +# 4. light mode: model ceiling STAYS armed +SID = "fbdisc-sonnet-" + RUN_TAG +subprocess.run([sys.executable, INJ], input=json.dumps( + {"cwd": tempfile.mkdtemp(prefix="fbseed_"), "session_id": SID, + "model": "claude-sonnet-5"}), capture_output=True, text=True) +d = proj(ledger="MODE: light\n- [ ] 1. card\n") +check("light/ceiling-stays", run(SPAWN, {"cwd": d, "session_id": SID, + "tool_name": "Agent", "tool_input": {"prompt": SMALL, "model": "opus"}}), 2) + +# 5. full mode unaffected (regression) +d = proj(ledger="MODE: full\n- [ ] 1. open card\n") +check("light/full-still-blocks", run(CLOSE, {"cwd": d}), 2) + +# ---- structural fail-streak ---- +FAIL_RESP = {"stdout": "", "stderr": "boom", "exitCode": 1} +OK_RESP = {"stdout": "ok", "stderr": "", "exitCode": 0} + +def hit(d, sid, resp): + return run_out(STREAK, {"cwd": d, "session_id": sid, "tool_name": "Bash", + "tool_input": {"command": "make test"}, + "tool_response": resp}) + +# 1. failures 1-5: exit 0 (advisory at 3); failure 6+: exit 2 (structural) +d = proj(ledger="- [ ] 1. card\n") +sid = "fbdisc-streak-1-" + RUN_TAG +results = [hit(d, sid, FAIL_RESP) for _ in range(7)] +check("streak6/advisory-under-6", all(rc == 0 for rc, _, _ in results[:5]), True) +check("streak6/advisory-at-3", "attribution ladder" in results[2][1], True) +check("streak6/hard-at-6", results[5][0], 2) +check("streak6/stays-hard-past-6", results[6][0], 2) +check("streak6/demands-tried-note", "-- tried:" in results[5][2], True) + +# 2. writing a `-- tried:` distillation into the ledger resets the streak +with open(os.path.join(d, ".fable", "LEDGER.md"), "a") as f: + f.write("- [ ] 1b. card -- tried: ruled out cache; test asserts wrong port\n") +rc, _, _ = hit(d, sid, FAIL_RESP) +check("streak6/tried-note-resets", rc, 0) + +# 3. success still resets (regression) +sid2 = "fbdisc-streak-2-" + RUN_TAG +d2 = proj(ledger="- [ ] 1. card\n") +for _ in range(6): + hit(d2, sid2, FAIL_RESP) +hit(d2, sid2, OK_RESP) +rc, _, _ = hit(d2, sid2, FAIL_RESP) +check("streak6/success-resets", rc, 0) + +# 4. PAUSED still disables the streak guard (regression) +d3 = proj(ledger="- [ ] 1. c\nPAUSED: side work\n") +sid3 = "fbdisc-streak-3-" + RUN_TAG +outs = [hit(d3, sid3, FAIL_RESP) for _ in range(7)] +check("streak6/paused-off", all(rc == 0 for rc, _, _ in outs), True) + +for d in tmps: + shutil.rmtree(d, ignore_errors=True) + +print("\n%d passed, %d failed" % (passed, failed)) +sys.exit(1 if failed else 0)