From c36a0d2d37332b55d6f65447b7a15ae73ea2ba04 Mon Sep 17 00:00:00 2001 From: Nibbler1250 Date: Mon, 3 Aug 2026 10:13:58 -0400 Subject: [PATCH] fix(tuner): wire the research-scout benchmark feeder into model_routing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The model_routing subject exposes an injectable benchmarkProvider (the "research scout" seam) but its default is `async () => []`, and at composition (registerWisecronSubjects) only the dispatchReader was wired — the benchmarkProvider was left at the empty default. So the benchmark reroute had no external evidence and could never surface a new-model reroute: the benchmark-side twin of the obs=0 dispatchReader gap. The scout itself (cache-first getModelBenchmarks over the Artificial Analysis free tier + enrichWithAnthropicCoding) was already built and tested; it was simply never injected. - add makeBenchmarkProvider() beside makeModeDispatchReader in observation-readers.ts (the runtime seam): cache-first fetch, ensures the cache dir exists, enriches Claude coding gaps from the Anthropic seed, graceful [] on missing key/outage, fetchImpl/nowMs injectable for tests. - inject it at ModelRoutingSubject construction; cache path + TTL are config-overridable (benchmark_cache_path / benchmark_ttl_ms). - the subject stays pure (never fetches the web itself); the runtime composition injects the feeder, per the governance rule in the subject. - tests: fetch+enrich, cache-dir creation, cache-first reuse, empty-key short-circuit (all hermetic). --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- .../wisecron/observation-readers.test.ts | 118 ++++++++++++++++++ src/tuner/wisecron/index.ts | 10 +- src/tuner/wisecron/observation-readers.ts | 61 ++++++++- 5 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 src/tuner/__tests__/wisecron/observation-readers.test.ts diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 409505f3..5755a171 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "name": "claudeclaw-plus", "source": "./", "description": "ClaudeClaw+ — governance, orchestration, persistent memory, and hardened web UI for Claude Code daemons. Sister project to moazbuilds/claudeclaw.", - "version": "2.2.217", + "version": "2.2.218", "keywords": [ "cron", "heartbeat", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5afb4fb5..0632ccef 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { "name": "claudeclaw-plus", - "version": "2.2.217", + "version": "2.2.218", "description": "ClaudeClaw+ — governance, orchestration, persistent memory, and hardened web UI for Claude Code daemons. Sister project to moazbuilds/claudeclaw." } diff --git a/src/tuner/__tests__/wisecron/observation-readers.test.ts b/src/tuner/__tests__/wisecron/observation-readers.test.ts new file mode 100644 index 00000000..39e4b2d0 --- /dev/null +++ b/src/tuner/__tests__/wisecron/observation-readers.test.ts @@ -0,0 +1,118 @@ +import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, existsSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { makeBenchmarkProvider } from "../../wisecron/observation-readers.js"; +import { ATTRIBUTION } from "../../subjects/model-routing-benchmarks.js"; + +const NOW = 1_700_000_000_000; + +// A shape-accurate Artificial Analysis /models/free response. The sonnet-5 row +// deliberately omits the coding eval so coding_index parses to null — the gap +// enrichWithAnthropicCoding is supposed to backfill from Anthropic's seed. +function aaResponse() { + return { + tier: "free", + data: [ + { + slug: "claude-sonnet-5", + name: "Claude Sonnet 5", + evaluations: { + artificial_analysis_intelligence_index: 71.5, + // no coding index → null → should be filled to 72.7 by the seed + artificial_analysis_agentic_index: 60.0, + }, + pricing: { price_1m_input_tokens: 3, price_1m_output_tokens: 15 }, + }, + ], + }; +} + +function mockFetch(body: unknown, ok = true, status = 200) { + const calls: Array<{ url: string; headers?: Record }> = []; + const impl = async (url: string, init?: { headers?: Record }) => { + calls.push({ url, headers: init?.headers }); + return { ok, status, json: async () => body }; + }; + return { impl, calls }; +} + +describe("makeBenchmarkProvider — research-scout feeder wiring", () => { + let dir: string; + let cachePath: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "bench-provider-")); + cachePath = join(dir, "cache.json"); + }); + afterEach(() => rmSync(dir, { recursive: true, force: true })); + + it("fetches published benchmarks and enriches Claude coding gaps from the Anthropic seed", async () => { + const { impl, calls } = mockFetch(aaResponse()); + const provider = makeBenchmarkProvider({ + cachePath, + apiKey: "test-key", + fetchImpl: impl, + nowMs: () => NOW, + }); + + const rows = await provider(["claude-sonnet-5"]); + + expect(calls.length).toBe(1); + const sonnet = rows.find((r) => r.model_id === "claude-sonnet-5"); + expect(sonnet).toBeDefined(); + expect(sonnet?.intelligence_index).toBe(71.5); + // null coding_index backfilled from ANTHROPIC_SWE_BENCH_VERIFIED + expect(sonnet?.coding_index).toBe(72.7); + // enrichment stamps the Anthropic source alongside AA attribution + expect(sonnet?.source).toContain(ATTRIBUTION); + expect(sonnet?.source.toLowerCase()).toContain("anthropic"); + }); + + it("creates the cache directory and persists the fetched benchmarks", async () => { + const nested = join(dir, "deep", "nested", "cache.json"); + const { impl } = mockFetch(aaResponse()); + const provider = makeBenchmarkProvider({ + cachePath: nested, + apiKey: "test-key", + fetchImpl: impl, + nowMs: () => NOW, + }); + + await provider(["claude-sonnet-5"]); + expect(existsSync(nested)).toBe(true); + const body = JSON.parse(readFileSync(nested, "utf8")); + expect(body.attribution).toBe(ATTRIBUTION); + expect(Array.isArray(body.benchmarks)).toBe(true); + }); + + it("serves the second call from cache without re-fetching (cache-first)", async () => { + const { impl, calls } = mockFetch(aaResponse()); + const provider = makeBenchmarkProvider({ + cachePath, + apiKey: "test-key", + fetchImpl: impl, + nowMs: () => NOW, + }); + + await provider(["claude-sonnet-5"]); + await provider(["claude-sonnet-5"]); + expect(calls.length).toBe(1); + }); + + it("is graceful: an empty API key short-circuits to [] rather than throwing or fetching", async () => { + // apiKey:"" forces the missing-key branch deterministically (fetchModelBenchmarks + // treats a falsy key as "no key" before any fetch), independent of the ambient + // ARTIFICIAL_ANALYSIS_API_KEY in the process env. + const { impl, calls } = mockFetch(aaResponse()); + const provider = makeBenchmarkProvider({ + cachePath, + apiKey: "", + fetchImpl: impl, + nowMs: () => NOW, + }); + const rows = await provider(["claude-sonnet-5"]); + expect(rows).toEqual([]); + expect(calls.length).toBe(0); + }); +}); diff --git a/src/tuner/wisecron/index.ts b/src/tuner/wisecron/index.ts index f38172cd..17df668d 100644 --- a/src/tuner/wisecron/index.ts +++ b/src/tuner/wisecron/index.ts @@ -38,7 +38,7 @@ import { SkillsSubject } from "../subjects/skills-subject.js"; // proactive content-patch face). The remaining wisecron subjects (cron, // claude_md, hook, prompt_template, agent) land in their own follow-up bricks // once the loop has earned its keep. -import { makeModeDispatchReader } from "./observation-readers.js"; +import { makeModeDispatchReader, makeBenchmarkProvider } from "./observation-readers.js"; export interface WisecronContext { db: WisecronStateDB; @@ -114,6 +114,14 @@ export function registerWisecronSubjects( dispatchReader: makeModeDispatchReader( cfg("model_routing").observation_log as string | undefined, ), + // Inject the research-scout feeder so the benchmark reroute has external + // evidence. The subject's default benchmarkProvider is `() => []` (never + // wired at composition), the direct cause of it never surfacing a + // new-model reroute — the benchmark-side twin of the dispatchReader gap. + benchmarkProvider: makeBenchmarkProvider({ + cachePath: cfg("model_routing").benchmark_cache_path as string | undefined, + ttlMs: cfg("model_routing").benchmark_ttl_ms as number | undefined, + }), }), ); diff --git a/src/tuner/wisecron/observation-readers.ts b/src/tuner/wisecron/observation-readers.ts index 89a82e5c..73662d34 100644 --- a/src/tuner/wisecron/observation-readers.ts +++ b/src/tuner/wisecron/observation-readers.ts @@ -16,11 +16,18 @@ * Wired in `registerWisecronSubjects`; each subject keeps its injectable seam, * so tests can still override with fixtures. */ -import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync, mkdirSync } from "node:fs"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { join, dirname } from "node:path"; import { DEFAULT_TOOL_CALL_LOG } from "../../observability/tool-call-sink.js"; import { DEFAULT_MODE_DISPATCH_LOG } from "../../governance/mode-dispatch-journal.js"; +import { + getModelBenchmarks, + DEFAULT_TTL_MS, + type ModelBenchmark, + type FetchBenchmarksOptions, +} from "../subjects/model-routing-benchmarks.js"; +import { enrichWithAnthropicCoding } from "../subjects/anthropic-benchmarks.js"; function expandHome(p: string): string { return p.startsWith("~") ? p.replace(/^~/, homedir()) : p; @@ -122,6 +129,56 @@ export function makeModeDispatchReader( }; } +/** + * `ModelRoutingSubject.benchmarkProvider` factory — the "research scout" feeder. + * + * The subject is deliberately pure: it NEVER fetches the web itself, and its + * default provider is `async () => []`, so with nothing injected the benchmark + * reroute has no external evidence and proposes nothing (safe, but inert). That + * default was never overridden at composition, which is the direct cause of the + * routing subject never surfacing a new-model reroute — the twin of the obs=0 + * gap `makeModeDispatchReader` closes on the cost side. + * + * This factory lives at the runtime seam (not inside the subject) and injects + * the already-built scout: cache-first `getModelBenchmarks` (Artificial Analysis + * free tier, `ARTIFICIAL_ANALYSIS_API_KEY`), then `enrichWithAnthropicCoding` to + * fill Claude coding gaps the AA free tier leaves null. Graceful throughout — a + * missing key or benchmark outage yields `[]`, never a throw, so the proactive + * loop is never stalled. `fetchImpl`/`nowMs` stay injectable for tests. + */ +export function makeBenchmarkProvider( + opts: { + cachePath?: string; + ttlMs?: number; + apiKey?: string; + fetchImpl?: FetchBenchmarksOptions["fetchImpl"]; + nowMs?: () => number; + } = {}, +): (models: string[]) => Promise { + const cachePath = expandHome( + opts.cachePath ?? join(homedir(), ".claude", "tuner", "model-benchmarks-cache.json"), + ); + const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS; + const now = opts.nowMs ?? (() => Date.now()); + return async (models: string[]) => { + // writeBenchmarkCache uses writeFileSync without mkdir; ensure the dir once. + try { + mkdirSync(dirname(cachePath), { recursive: true }); + } catch { + /* best-effort; a cache-dir failure degrades to fetch-every-time, not a throw */ + } + const rows = await getModelBenchmarks({ + models, + cachePath, + ttlMs, + apiKey: opts.apiKey, + fetchImpl: opts.fetchImpl, + nowMs: now(), + }); + return enrichWithAnthropicCoding(rows); + }; +} + /** Shape of HookSubject's HookLogEntry (kept structural — the subject's interface * is internal; this matches it field-for-field). */ interface HookLogEntryShape {