Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions hooks/dispatch-counter.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,29 @@
// Log lives in <config>/model-routing/dispatches.jsonl and self-prunes to 30d.

import { appendFileSync, closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, writeFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { homedir } from "node:os";

const DAY_MS = 24 * 60 * 60 * 1000;
const RETENTION_MS = 30 * DAY_MS;

// Stamped on every report so a pasted screenshot answers "which version wrote
// this" without a second round trip - the question that costs the most time
// when someone reports a report that looks wrong. Read from the manifest next
// to this script rather than CLAUDE_PLUGIN_ROOT, which the hook sets but a
// direct `node hooks/dispatch-counter.mjs report` does not. Unreadable manifest
// degrades to an unlabelled header; a version stamp must never break a report.
const PLUGIN_VERSION = (() => {
try {
const here = dirname(fileURLToPath(import.meta.url));
return JSON.parse(readFileSync(join(here, "..", ".claude-plugin", "plugin.json"), "utf-8")).version ?? null;
} catch {
return null;
}
})();
const versionSuffix = PLUGIN_VERSION ? ` - model-routing ${PLUGIN_VERSION}` : "";

// Window flags: --days N (size, default 7) and --ago M (shift back M days).
// Bad values fall back to the default rather than erroring - a stats tool
// must never be harder to run than the thing it measures.
Expand Down Expand Up @@ -369,7 +386,9 @@ if (process.argv[2] === "stats" || process.argv[2] === "report") {
// report is indistinguishable from a broken node/shell run.
process.stdout.write(process.argv[2] === "stats"
? `routed-down: no data (${winLabel})`
: `No dispatches logged in the window (${winLabel}).\nLog: ${dataFile()} - history kept 30 days.\nEntries appear after the first Agent dispatch once the plugin's PostToolUse hook is active (plugin enabled + session restarted).`);
// The version belongs here most of all: an empty report is the one people
// screenshot, and "which version" is the first thing it has to answer.
: `No dispatches logged in the window (${winLabel})${versionSuffix}.\nLog: ${dataFile()} - history kept 30 days.\nEntries appear after the first Agent dispatch once the plugin's PostToolUse hook is active (plugin enabled + session restarted).`);
process.exit(0);
}
// Per-entry verdict: down / at / up / unknown. "up" = the effective model
Expand Down Expand Up @@ -537,7 +556,7 @@ if (process.argv[2] === "stats" || process.argv[2] === "report") {
const pct = comparable ? Math.round((down.length / comparable) * 100) : 0;
const section = (title, rows2) => rows2.length ? ["", title, ...rows2] : [];
const lines = [
`Model routing report - ${winLabel}`,
`Model routing report - ${winLabel}${versionSuffix}`,
"",
`${down.length} of ${comparable}${unknownCount ? " comparable" : ""} dispatches (${pct}%) ran on a cheaper model than the session${unknownCount ? ` - ${unknownCount} not tier-comparable excluded` : ""}${todayPart ? ` (${todayPart.replace(" · ", "")})` : ""}.`,
...(upCount ? [`${upCount} ran ABOVE the session tier - a pin above the session model, uncapped; pins are ceilings only when the dispatch passes model=<session>.`] : []),
Expand Down Expand Up @@ -785,7 +804,7 @@ if (process.argv[2] === "tokens") {
const mainNote = mainVolTotal
? `\n\nMain sessions in this window: ${fmtN(mainVolTotal)} across ${mainSessions} sessions, with no subagent volume counted against them.`
: "";
process.stdout.write(`No subagent transcripts found under ${projRoot} (${winLabel}).\nToken stats read Claude Code agent-*.jsonl transcript files; they appear after subagent dispatches. If your config lives elsewhere, set CLAUDE_CONFIG_DIR.${mainNote}`);
process.stdout.write(`No subagent transcripts found under ${projRoot} (${winLabel})${versionSuffix}.\nToken stats read Claude Code agent-*.jsonl transcript files; they appear after subagent dispatches. If your config lives elsewhere, set CLAUDE_CONFIG_DIR.${mainNote}`);
process.exit(0);
}
const rows = [...perModel.entries()].map(([m, s]) => ({ m, vol: volOf(s), ...s }))
Expand All @@ -799,7 +818,7 @@ if (process.argv[2] === "tokens") {
const bar = (v) => "#".repeat(Math.max(1, Math.round((v / total) * 24)));
const sessionRows = [...perSession.entries()].sort((a, b) => b[1].vol - a[1].vol);
const out = [
`Subagent token volume - ${winLabel} (input + cache):`,
`Subagent token volume - ${winLabel} (input + cache)${versionSuffix}:`,
"",
`${fmtN(downTotal)} of ${fmtN(total - unknownVol)} ${unknownVol ? "comparable " : ""}tokens (${Math.round((downTotal / comparableVol) * 100)}%) processed on a cheaper model than their session - judged per session (fable/opus days both count fairly).`,
// A --session filter selects the sessions with the most room to route down,
Expand Down
50 changes: 50 additions & 0 deletions hooks/dispatch-counter.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1227,3 +1227,53 @@ test("a model in the tier table also has an effort row", () => {
assert.equal(e.effort, "high");
assert.equal(e.effortFrom, "default");
});

const manifestVersion = () =>
JSON.parse(
readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", ".claude-plugin", "plugin.json"), "utf-8")
).version;

test("both reports stamp the plugin version from the manifest", () => {
const cfg = freshConfigDir();
writeLog(cfg, [{ ts: Date.now(), agent: "general-purpose", model: "sonnet", session: "claude-opus-5" }]);
// Read the version rather than hardcoding it: this asserts the WIRING - that
// the script resolves the manifest next to itself - and cannot go stale at the
// next release the way a literal would.
const stamp = "model-routing " + manifestVersion();
const dir = join(cfg, "projects", "proj", "sess-1", "subagents");
mkdirSync(dir, { recursive: true });
writeFileSync(join(cfg, "projects", "proj", "sess-1.jsonl"), usageLine("claude-opus-5", 1) + "\n");
writeFileSync(join(dir, "agent-a.jsonl"), usageLine("claude-sonnet-5", 1000) + "\n");
try {
assert.ok(run(["report"], cfg).split("\n")[0].endsWith(stamp), "report header must end with the version stamp");
assert.ok(run(["tokens"], cfg).split("\n")[0].endsWith(stamp + ":"), "tokens header must end with the version stamp");
// The three empty-data paths matter most: they are what people screenshot
// when something looks wrong, and "which version wrote this" is the first
// question those reports have to answer.
assert.ok(run(["report", "--days", "1", "--ago", "9000"], cfg).includes(stamp), "empty report must carry it");
const noTranscripts = run(["tokens"], freshConfigDir());
assert.ok(noTranscripts.includes(stamp), "the no-transcripts message must carry it too");
} finally { rmSync(cfg, { recursive: true, force: true }); }
});

test("an unreadable manifest costs the stamp, not the report", () => {
// The script copied away from its manifest: the version cannot be resolved,
// and the report must still print - not crash, and not say "undefined".
const dir = mkdtempSync(join(tmpdir(), "mr-noman-"));
const script = join(dir, "dispatch-counter.mjs");
writeFileSync(script, readFileSync(join(dirname(fileURLToPath(import.meta.url)), "dispatch-counter.mjs")));
const cfg = freshConfigDir();
writeLog(cfg, [{ ts: Date.now(), agent: "general-purpose", model: "sonnet", session: "claude-opus-5" }]);
try {
const out = execFileSync(process.execPath, [script, "report"], {
env: { ...process.env, CLAUDE_CONFIG_DIR: cfg, CLAUDE_CODE_SUBAGENT_MODEL: "", CLAUDE_CODE_EFFORT_LEVEL: "" },
cwd: cfg,
encoding: "utf-8",
});
assert.match(out, /^Model routing report - 7d\n/);
assert.doesNotMatch(out, /undefined|null/);
} finally {
rmSync(cfg, { recursive: true, force: true });
rmSync(dir, { recursive: true, force: true });
}
});
Loading