Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions .claude/hooks/ask-routing-deferral-detector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,7 @@ import type { TranscriptLine } from "./transcript";
import { logCalibrationRecord, logEvaluationRecord } from "./dispatcher";
import type { DispatchContext, GuardOutcome } from "./registry";
import { elideQuotedContexts, elideDoubleQuotedSpans } from "./elision";
import {
CAPTURE_SCHEMA_FIELD,
CAPTURE_SCHEMA_VERSION,
extractMatchContext,
} from "./judged-input-capture";
import { captureFields, extractMatchContext } from "./judged-input-capture";
import { createHash } from "node:crypto";
import { cappedEvidenceLines, truncateToRenderedLength } from "./guard-feedback-format";
import { STOP_INJECTED_OVERLAP_FAMILY, overlapTurnKey, readFlagged } from "./turn-end-scan-store";
Expand Down Expand Up @@ -2017,7 +2013,28 @@ export async function run(
timestamp: new Date().toISOString(),
session_id: input.session_id,
injection_enabled: INJECTION_ENABLED,
[CAPTURE_SCHEMA_FIELD]: CAPTURE_SCHEMA_VERSION,
// mt#3866: stamps the capture marker AND the distinct-fire digest
// together. Before this, all 58 records in the live window carried the
// marker and no identifier, so four byte-identical records could not be
// told apart from four genuine emissions of one sentence.
...captureFields(assistantText),
// mt#3866 SC1's second half — "where the writer has it, a turn anchor".
// This path HAS one: the dispatcher resolves `recordedAnchor` once per
// invocation, and its `turnKey` is the opening prompt line's uuid.
//
// Stamped BESIDE the digest rather than instead of it, because the two
// answer different questions and only one is universal. The digest
// answers "same TEXT" and is available on every path; `turn_key` answers
// "same TURN", which is strictly what the ambiguity was about — two
// genuinely distinct turns emitting the identical sentence hash the same
// and would group as one, a limitation `captureFields`' own docblock
// names. The sweep still groups on the digest (see `countDistinctFires`)
// because a window mixing records with and without a turn key would
// split one turn across two grouping keys; this field is here for a
// reader or a replay that wants the finer answer.
...(ctx.recordedAnchor?.turnKey !== undefined
? { turn_key: ctx.recordedAnchor.turnKey }
: {}),
matches: calibrationMatches(matches),
suppressionReasons,
// ADR-024's degraded MARKER. Present only when a nomination was attempted
Expand Down Expand Up @@ -2165,7 +2182,9 @@ export async function main(): Promise<void> {
timestamp: new Date().toISOString(),
session_id: input.session_id,
injection_enabled: INJECTION_ENABLED,
[CAPTURE_SCHEMA_FIELD]: CAPTURE_SCHEMA_VERSION,
// mt#3866 — see the sibling site in `run()` for why marker and digest are
// stamped by one call.
...captureFields(assistantText),
matches: calibrationMatches(matches),
suppressionReasons,
...(settledRung2.degradedReason !== undefined
Expand Down
79 changes: 79 additions & 0 deletions .claude/hooks/judged-input-capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -291,3 +291,82 @@ export function captureArtifact(
truncated: text.length > maxChars,
};
}

// ---------------------------------------------------------------------------
// Record identity — the distinct-fire question (mt#3866)
// ---------------------------------------------------------------------------

/**
* The record key a judged-text digest is written under.
*
* Named to match `retrospective-trigger-scanner.ts`, which has written exactly
* this field since mt#3821 and was, before mt#3866, the ONLY calibration writer
* that did.
*/
export const JUDGED_TEXT_HASH_FIELD = "judged_text_hash";

/**
* The fields a calibration writer stamps to make its record both AUDITABLE and
* IDENTIFIABLE — spread into the record instead of writing the marker alone.
*
* ## Why this exists rather than two independent stamps (mt#3866)
*
* {@link CAPTURE_SCHEMA_FIELD} answers *"can this record's judged text be
* re-read?"*. It does NOT answer *"is this record the same FIRE as that one?"*,
* and nothing in its shape said so — which let a writer stamp the marker while
* carrying no identifier at all.
*
* That was not hypothetical. Measured 2026-09-05 on
* `ask-routing-deferral-calibration.jsonl`: all 58 records carried
* `captureSchema`, and the complete top-level key set was `timestamp,
* session_id, injection_enabled, captureSchema, matches, suppressionReasons` —
* no digest anywhere. Four records on 2026-09-04 carried a byte-identical
* `matches[].context` and could not be told apart from four genuine emissions
* of the same sentence, which bounded a precision figure to a range in
* mt#5004's gap report.
*
* The two capture helpers in this module are why the gap was invisible:
* {@link extractMatchContext} returns a bounded STRING and
* {@link captureArtifact} returns `{ excerpt, hash }`. A writer taking the
* first route gets re-readability with no identity, and the marker reads the
* same either way.
*
* **So the coupling is the fix.** A writer that stamps the marker through this
* function cannot omit the digest, because there is no longer a way to spell
* one without the other.
*
* ## What the digest can and cannot settle
*
* It answers "same TEXT", which is the discriminator the ambiguity actually
* needed: a 2,675-character message written three times across 3.5 hours would
* not hash identically, so identical digests in one session mean one message
* judged repeatedly (the route `## Evidence 2026-08-16` on mt#3866 took, by
* borrowing a sibling stream's digest).
*
* It does NOT answer "same TURN". Two genuinely distinct turns that emit the
* identical sentence hash identically and will group as one — deliberately
* accepted here, because the alternative needs a turn identifier the hook does
* not have at write time, and because collapsing a repeated sentence is the
* safer error for a rate whose denominator this feeds. A reader wanting turn
* identity should join the anchor stream, not this field.
*/
export function captureFields(judgedText: string): Record<string, unknown> {
return {
[CAPTURE_SCHEMA_FIELD]: CAPTURE_SCHEMA_VERSION,
[JUDGED_TEXT_HASH_FIELD]: hashJudgedText(judgedText),
};
}

/**
* Read a record's distinct-fire digest, or `undefined` when it carries none.
*
* `undefined` means **un-groupable**, and a caller must not read it as
* "distinct". A record written before mt#3866 has no digest, so treating
* absence as distinctness would silently re-create the over-count this field
* exists to remove — the same discipline {@link hasJudgedInputCapture} states
* for its own `false`.
*/
export function getJudgedTextHash(record: Record<string, unknown>): string | undefined {
const value = record[JUDGED_TEXT_HASH_FIELD];
return typeof value === "string" && value.length > 0 ? value : undefined;
}
18 changes: 12 additions & 6 deletions .claude/hooks/operator-deferral-detector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,7 @@ import type { SpecTextRead } from "./authored-spec-text";
import type { DispatchContext, GuardOutcome } from "./registry";
import { logCalibrationRecord, logEvaluationRecord } from "./dispatcher";
import { elideQuotedContexts, elideDoubleQuotedSpans } from "./elision";
import {
CAPTURE_SCHEMA_FIELD,
CAPTURE_SCHEMA_VERSION,
extractMatchContext,
} from "./judged-input-capture";
import { captureFields, extractMatchContext } from "./judged-input-capture";
// Surrogate-pair-safe truncation — the matched text is arbitrary assistant
// prose / operator-authored option labels, so a raw `.slice(0, N)` can split
// an emoji. Same cross-tree import the standalone parallel-work guard uses.
Expand Down Expand Up @@ -1570,7 +1566,17 @@ export function buildCalibrationRecord(
// hit; `context` carries the surrounding prose that used to occupy `phrase`.
// Both, because the axis needs the first to be meaningful and a human
// reviewer needs the second to classify the fire at all.
[CAPTURE_SCHEMA_FIELD]: CAPTURE_SCHEMA_VERSION,
// mt#3866 (PR #3656 R2): the marker and the distinct-fire digest are
// stamped by ONE call, so this writer cannot claim capture without also
// carrying an identity. `turnText` is the judged text — the same string
// `deferralOverlap` is derived from three lines above.
//
// `turnText` is optional on this path and empty for a tool-call-only fire
// (surface E), so the digest is over `""` there. That is deliberate and
// matches `captureFields`' own contract: an empty judged text is still a
// judged text, and returning no digest would let a writer opt out of
// identity by passing nothing — which is the separability this fixes.
...captureFields(turnText ?? ""),
matches: matches.map((m) => ({
category: m.surface,
phrase: m.matchedPhrase,
Expand Down
28 changes: 23 additions & 5 deletions .claude/hooks/pre-narration-detector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ import { safeTruncate } from "@minsky/shared/safe-truncate";
// change than this task's evidence supports.
import { elideDoubleQuotedSpans } from "./elision";
import {
CAPTURE_SCHEMA_FIELD,
CAPTURE_SCHEMA_VERSION,
captureFields,
extractMatchContext,
MATCH_CONTEXT_MAX_CHARS,
} from "./judged-input-capture";
Expand Down Expand Up @@ -484,6 +483,22 @@ export interface PreNarrationDetection {
matches: ClaimMatch[];
/** Backed claims — detected, then suppressed. Recorded, never injected. */
suppressed: SuppressedClaimMatch[];
/**
* The assistant text this detection was computed over (mt#3866).
*
* Carried on the RESULT rather than threaded as a third parameter to
* `buildPreNarrationRecord`, which would have touched nine call sites for a
* value the detection already had in hand. It is genuinely part of what was
* detected: `detectPreNarrationWithSuppression` derives it internally with
* `extractAssistantText` and every match's offsets address it.
*
* `""` on the no-text early return, and that case is real — a turn carrying
* only tool calls extracts to the empty string. The digest is then over `""`,
* which `captureFields` treats as a judged text like any other; returning no
* digest there would let a writer opt out of identity by having nothing to
* say, which is the separability mt#3866 exists to remove.
*/
judgedText: string;
}

/**
Expand Down Expand Up @@ -673,7 +688,7 @@ export function detectPreNarrationWithSuppression(
evidencePrNumbers?: ReadonlyMap<string, ReadonlySet<number>>
): PreNarrationDetection {
const rawText = extractAssistantText(turnLines);
if (!rawText) return { matches: [], suppressed: [] };
if (!rawText) return { matches: [], suppressed: [], judgedText: "" };

// Double-quoted prose elided AFTER the markdown pass (mt#3864 class 6), so
// quotes inside a code span are already blanked and cannot confuse pairing —
Expand Down Expand Up @@ -747,7 +762,7 @@ export function detectPreNarrationWithSuppression(
}
matches.push(matched);
}
return { matches, suppressed };
return { matches, suppressed, judgedText: rawText };
}

/**
Expand Down Expand Up @@ -788,7 +803,10 @@ export function buildPreNarrationRecord(
// marker says so explicitly, so a corpus-wide auditability check reads the
// same field everywhere instead of special-casing the surfaces that shipped
// capture before the marker existed.
[CAPTURE_SCHEMA_FIELD]: CAPTURE_SCHEMA_VERSION,
// mt#3866 (PR #3656 R2): marker and distinct-fire digest stamped by ONE
// call, so this writer cannot claim capture without carrying an identity.
// The judged text rides on the detection — see `PreNarrationDetection`.
...captureFields(detection.judgedText),
matches: [
...detection.matches.map((m) => ({
category: m.category,
Expand Down
27 changes: 27 additions & 0 deletions .claude/skills/calibration-review/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,33 @@ never reached the operator and is NOT a fire for cadence purposes),
unreviewed matches), and `openAskId` (mt#2659 — set when a prior pass filed a
disposition Ask for this log that hasn't been resolved yet).

**`distinctFiresSinceLastReview` and `ungroupableSinceLastReview` bound the fire
count to a RANGE — read them as a pair, never singly (mt#3866).** `firesSinceLastReview`
is a RECORD count, and a record is not a fire: one message re-scanned across
several turns writes several records. Measured on `ask-routing-deferral`, 6 of
its duplicate groups resolved to 5 re-scans and 1 genuinely-distinct pair, so a
raw count over such a window overstates by roughly 18%.

- **`distinctFiresSinceLastReview`** — records grouped by `(session_id,
judged-text digest)`. Session-scoped deliberately: the identical closing
sentence in two conversations is two fires, and a digest-only key would
collapse them.
- **`ungroupableSinceLastReview`** — records carrying NO digest, i.e. written
before mt#3866 or by a writer that has not adopted `captureFields`. These are
in NEITHER column, because an un-groupable record is not evidence of a
distinct fire and not evidence of a duplicate.

So `distinct: 3, ungroupable: 8` means the true count is somewhere in `[3, 11]`
and only the first number is measured. **Quote the range, not a point estimate**,
whenever `ungroupableSinceLastReview` is non-zero — and note that on a
pre-mt#3866 window `distinctFires` is 0 and says nothing at all.

To resolve a historical group that carries no digest, run
`bun scripts/resolve-calibration-duplicate-groups.ts <detector>` — it borrows a
digest from a sibling evaluation stream on `(session_id, timestamp ± 1s)`. Only
`causal-premise` hashes its judged input today, so `unresolvable` is the common
verdict and means "no oracle covered that turn", never "these are distinct".

### Step 1a — Reconcile any already-open disposition ask (mt#2659)

Before doing new FP-classification work, check any log whose `openAskId` is
Expand Down
76 changes: 76 additions & 0 deletions .minsky/hooks/ask-routing-deferral-detector.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import type { NominationDeps } from "../../packages/domain/src/detectors/embeddi
import type { TranscriptLine } from "./transcript";
import type { ClaudeHookInput } from "./types";
import type { DispatchContext } from "./registry";
import { JUDGED_TEXT_HASH_FIELD } from "./judged-input-capture";

const PRINCIPAL_RESERVED = "principal-reserved" as const;
const DEFERRAL_MENU = "deferral-menu" as const;
Expand Down Expand Up @@ -2090,3 +2091,78 @@ describe("mt#4702 — an escape hatch on a stated commitment is an override, not
expect(findOfferShape("I'll proceed unless you'd rather I wait")).toBeNull();
});
});

// ---------------------------------------------------------------------------
// mt#3866 — record identity: the digest, and the turn key where one exists
// ---------------------------------------------------------------------------

describe("mt#3866 — run() stamps a distinct-fire identity", () => {
/** A turn that fires the principal-reserved class, so a record is written. */
const DEFERRAL_TURN = "The tier naming needs your call before anything ships.";

function calibrationOf(outcome: Awaited<ReturnType<typeof run>>): Record<string, unknown> {
expect(outcome?.calibration).toBeDefined();
return outcome?.calibration as unknown as Record<string, unknown>;
}

test("the record carries BOTH the capture marker and the judged-text digest", async () => {
const lines = [makeRunUserLine(), makeRunAssistantLine(DEFERRAL_TURN), makeRunUserLine()];
const cal = calibrationOf(await run(RUN_HOOK_INPUT, makeCtx(lines)));

// The measured pre-mt#3866 shape was the marker WITHOUT the digest, on all
// 58 records in the live window. Both halves are asserted so a regression
// that drops one is visible.
expect(cal["captureSchema"]).toBe(1);
expect(typeof cal[JUDGED_TEXT_HASH_FIELD]).toBe("string");
});

test("two runs over the SAME turn text produce the same digest", async () => {
const lines = [makeRunUserLine(), makeRunAssistantLine(DEFERRAL_TURN), makeRunUserLine()];
const first = calibrationOf(await run(RUN_HOOK_INPUT, makeCtx(lines)));
const second = calibrationOf(await run(RUN_HOOK_INPUT, makeCtx(lines)));

// This is the ambiguity the task exists to remove, at the write side: a
// re-scan of one message now announces itself.
expect(second[JUDGED_TEXT_HASH_FIELD]).toBe(first[JUDGED_TEXT_HASH_FIELD] as string);
});

test("a DIFFERENT turn produces a different digest — the control", async () => {
const a = calibrationOf(
await run(
RUN_HOOK_INPUT,
makeCtx([makeRunUserLine(), makeRunAssistantLine(DEFERRAL_TURN), makeRunUserLine()])
)
);
const b = calibrationOf(
await run(
RUN_HOOK_INPUT,
makeCtx([
makeRunUserLine(),
makeRunAssistantLine("The panel copy needs your call before anything ships."),
makeRunUserLine(),
])
)
);
expect(b[JUDGED_TEXT_HASH_FIELD]).not.toBe(a[JUDGED_TEXT_HASH_FIELD] as string);
});

test("SC1's second half: a recorded turn anchor is stamped as `turn_key`", async () => {
const lines = [makeRunUserLine(), makeRunAssistantLine(DEFERRAL_TURN), makeRunUserLine()];
const cal = calibrationOf(
await run(RUN_HOOK_INPUT, {
...makeCtx(lines),
recordedAnchor: { turnKey: "turn-uuid-1234", lastAssistantMessage: DEFERRAL_TURN },
})
);
expect(cal["turn_key"]).toBe("turn-uuid-1234");
});

test("no anchor -> the key is ABSENT, not an empty string", async () => {
// An empty string would group every anchorless record together, which is
// the opposite error from the over-count and equally unfounded. `makeCtx`
// sets no `recordedAnchor`, which is the standalone-path shape.
const lines = [makeRunUserLine(), makeRunAssistantLine(DEFERRAL_TURN), makeRunUserLine()];
const cal = calibrationOf(await run(RUN_HOOK_INPUT, makeCtx(lines)));
expect("turn_key" in cal).toBe(false);
});
});
Loading
Loading