diff --git a/docs/JOURNAL.md b/docs/JOURNAL.md index b9eae8f..2b31cd9 100644 --- a/docs/JOURNAL.md +++ b/docs/JOURNAL.md @@ -19,6 +19,39 @@ or files, and the "so-what" for future readers. --- +## 2026-07-18 — Corpus synthesis re-paid for every map call: a per-article extract cache + +**Tags:** design + +The corpus map/reduce re-ran the ENTIRE map every "Analyze corpus" — +one LLM call per member, every time — because nothing persisted the +per-article extracts (`xray-audits` stored only `runs`/`predictions`/ +`resolutions`/`case-briefs`). Three runs over a 129-member case paid +for ~3× the map, which is the bulk of the cost (129 map calls vs 1 +reduce). The audit RUNS cache (keyed by article hash) is a different +feature; the corpus map path was uncached. + +**Fix:** a `corpus-extracts` store (xray-audits v3), keyed by +`corpusExtractKey` — a SHA-256 over the exact map inputs (sent text + +claim digest + case framing + `CORPUS_PROMPT_VERSION`). The runner +checks the cache before the confirm (so the spend preview says "N of M +cached — reused for free"), short-circuits cached members with no LLM +call, and persists each miss. **Invalidation falls out of the key:** a +body edit changes the text; a Suggest pass changes the claim digest (so +that article — and only that article — re-maps, which is correct); a +prompt bump changes the version. The reduce still runs every time (one +call, must see the full current extract set). After run 1, a re-run +over an unchanged corpus costs just the reduce; adding one article +costs one map call, not 129. + +Deliberately keyed on the *inputs*, not the article hash alone: the map +prompt carries each article's claim digest, so keying on the hash would +serve a stale extract after claims were added. The cache rides the +export-included audit DB (a hit is a dollar saved, worth carrying +across a restore); it is reconcilable and never auto-dropped. No wire +kind, no prompt change — pure cost reuse, identical outputs, so brief +quality is unchanged. + ## 2026-07-18 — Case synthesis saw ~2 of ~1,900 claims: membership was a union, claim-attachment wasn't **Tags:** bug, design diff --git a/src/portal/synthesis-block.js b/src/portal/synthesis-block.js index b910028..676df0f 100644 --- a/src/portal/synthesis-block.js +++ b/src/portal/synthesis-block.js @@ -17,11 +17,11 @@ import { el, truncate } from './dom.js'; import { Utils } from '../shared/utils.js'; import { AssessmentModel } from '../shared/assessment-model.js'; import { orchestrateModuleRuns } from '../shared/audit/run-orchestrator.js'; -import { getCaseBrief, saveCaseBrief } from '../shared/audit/audit-cache.js'; +import { getCaseBrief, saveCaseBrief, getCorpusExtract, saveCorpusExtract } from '../shared/audit/audit-cache.js'; import { createGroundingIndex } from '../shared/quote-grounding.js'; import { CORPUS_PROMPT_VERSION } from '../shared/corpus-prompts.js'; import { - buildMemberUnits, corpusInputHash, digestDossier, + buildMemberUnits, corpusInputHash, corpusExtractKey, digestDossier, validateCorpusExtract, validateCaseBrief, groundCaseBrief, filterProposals } from '../shared/case-synthesis.js'; import { renderProposals } from './synthesis-review.js'; @@ -343,18 +343,50 @@ export function renderSynthesisBlock(host, { data, dossier, callbacks = {} }) { runBtn.addEventListener('click', async () => { if (members.length === 0) { status.textContent = 'No archived member articles to analyze.'; return; } - const approxChars = members.reduce((a, m) => a + m.text.length, 0); - if (!confirm(`Analyze this corpus with the LLM?\n\n` - + `This sends ${members.length} member article${members.length === 1 ? '' : 's'} ` - + `(~${Math.round(approxChars / 1000)}k characters) to Anthropic — one call per article, then one synthesis call.`)) return; - runBtn.disabled = true; + const caseName = data.case.name || ''; const scopeQuestion = (dossier.scope && dossier.scope.question) || ''; const unitById = {}; for (const m of members) unitById[m.article_hash] = m; + // The exact map request for a member, and its cache key. The + // request shape is the map's whole input; the key fingerprints + // it so an unchanged article's extract is reused for free. + const reqOf = (m) => ({ + member_id: m.article_hash, memberText: m.text, + memberMeta: { title: m.title, url: m.url }, + claimsDigest: m.claims.map((c) => `${c.id} — ${c.text}`).join('\n'), + caseName, scopeQuestion + }); + + // Cost preview: the map is the bulk of a synthesis, so a re-run + // over an unchanged corpus should send nothing but the reduce. + status.textContent = 'Checking cache…'; + const keyByHash = {}; + const cachedByHash = {}; + for (const m of members) { + const key = await corpusExtractKey(reqOf(m)); + keyByHash[m.article_hash] = key; + const hit = await getCorpusExtract(key).catch(() => null); + if (hit && hit.extract && validateCorpusExtract(hit.extract).ok) cachedByHash[m.article_hash] = hit; + } + status.textContent = ''; + + const toSend = members.filter((m) => !cachedByHash[m.article_hash]); + const cachedCount = members.length - toSend.length; + const approxChars = toSend.reduce((a, m) => a + m.text.length, 0); + if (!confirm(`Analyze this corpus with the LLM?\n\n` + + (cachedCount ? `${cachedCount} of ${members.length} article${members.length === 1 ? '' : 's'} cached — reused for free.\n` : '') + + `This sends ${toSend.length} article${toSend.length === 1 ? '' : 's'} ` + + `(~${Math.round(approxChars / 1000)}k characters) to Anthropic, then one synthesis call.`)) { + runBtn.disabled = false; + return; + } + // MAP — bounded pool over member ids (the audit-module pattern). + // A cached member short-circuits with no LLM call; a miss calls + // and then persists the extract keyed by its input fingerprint. status.textContent = `Analyzing 0/${members.length} articles…`; const { modules, failures } = await orchestrateModuleRuns({ moduleNames: members.map((m) => m.article_hash), @@ -363,16 +395,14 @@ export function renderSynthesisBlock(host, { data, dossier, callbacks = {} }) { if (p.phase === 'done') status.textContent = `Analyzing ${p.okCount}/${p.total} articles…`; }, send: async (id) => { - const u = unitById[id]; - const res = await sendMessage({ type: 'xray:llm:corpus-map', request: { - member_id: id, memberText: u.text, - memberMeta: { title: u.title, url: u.url }, - claimsDigest: u.claims.map((c) => `${c.id} — ${c.text}`).join('\n'), - caseName, scopeQuestion - } }); + const cached = cachedByHash[id]; + if (cached) return { ok: true, findings: cached.extract, model: cached.model }; + const res = await sendMessage({ type: 'xray:llm:corpus-map', request: reqOf(unitById[id]) }); if (!res || !res.ok) return { ...(res || {}), ok: false }; const v = validateCorpusExtract(res.extract); if (!v.ok) return { ok: false, error: 'invalid extract' }; + saveCorpusExtract({ key: keyByHash[id], extract: res.extract, model: res.model, cachedAt: Math.floor(Date.now() / 1000) }) + .catch((err) => Utils.error('saveCorpusExtract failed', err)); return { ok: true, findings: res.extract, model: res.model }; } }); @@ -421,6 +451,7 @@ export function renderSynthesisBlock(host, { data, dossier, callbacks = {} }) { inputHash: liveHash, model: reduce.model, promptVersion: CORPUS_PROMPT_VERSION, members: members.length, analyzed: extracts.length, failed: failures.length, + cached: cachedCount, usage: reduce.usage || null, triage: (prior && prior.triage) || {} }; @@ -430,7 +461,8 @@ export function renderSynthesisBlock(host, { data, dossier, callbacks = {} }) { const coverageNote = failures.length ? ` (${extracts.length} of ${members.length} members analyzed; ${failures.length} failed)` : ''; - status.textContent = `Done${coverageNote}.`; + const cacheNote = cachedCount ? ` — ${cachedCount} reused from cache` : ''; + status.textContent = `Done${coverageNote}${cacheNote}.`; runBtn.disabled = false; renderStored(record); }); diff --git a/src/shared/audit/audit-cache.js b/src/shared/audit/audit-cache.js index a779f11..409186f 100644 --- a/src/shared/audit/audit-cache.js +++ b/src/shared/audit/audit-cache.js @@ -12,7 +12,7 @@ import { Utils } from '../utils.js'; const DB_NAME = 'xray-audits'; -const DB_VERSION = 2; +const DB_VERSION = 3; const RUNS_STORE = 'runs'; const PREDICTIONS_STORE = 'predictions'; const RESOLUTIONS_STORE = 'resolutions'; @@ -20,6 +20,13 @@ const RESOLUTIONS_STORE = 'resolutions'; // brief costs an LLM map/reduce run), so it rides the same // export-included DB; keyed by the local case entity id. const CASE_BRIEFS_STORE = 'case-briefs'; +// Per-article MAP-stage extracts, keyed by a fingerprint of the exact +// map inputs (text + claims + prompt version). Lets a corpus re-run +// reuse an unchanged article's extract instead of re-paying for the +// LLM call — the map is the bulk of a synthesis's cost. Rides the same +// export-included DB so the cache survives a restore (a hit is a +// dollar saved); reconcilable, but never auto-dropped here. +const CORPUS_EXTRACTS_STORE = 'corpus-extracts'; function idb() { if (typeof indexedDB === 'undefined') { @@ -81,6 +88,13 @@ export function openAuditDb() { db.createObjectStore(CASE_BRIEFS_STORE, { keyPath: 'caseId' }); } } + + // v3 — the per-article MAP-extract cache. + if (oldVersion < 3) { + if (!db.objectStoreNames.contains(CORPUS_EXTRACTS_STORE)) { + db.createObjectStore(CORPUS_EXTRACTS_STORE, { keyPath: 'key' }); + } + } }; open.onsuccess = () => resolve(open.result); open.onerror = () => reject(open.error); @@ -172,16 +186,25 @@ export function getCaseBrief(caseId) { return get(CASE_BRIEFS_STORE, caseId); } export function deleteCaseBrief(caseId) { return remove(CASE_BRIEFS_STORE, caseId); } export function listCaseBriefs() { return getAll(CASE_BRIEFS_STORE); } +// --- corpus map-extract cache (map/reduce cost reuse) --------------------------- + +export function saveCorpusExtract(record) { return put(CORPUS_EXTRACTS_STORE, record); } +export function getCorpusExtract(key) { return get(CORPUS_EXTRACTS_STORE, key); } +export function deleteCorpusExtract(key) { return remove(CORPUS_EXTRACTS_STORE, key); } +export function listCorpusExtracts() { return getAll(CORPUS_EXTRACTS_STORE); } +export function countCorpusExtracts() { return countStore(CORPUS_EXTRACTS_STORE); } + // --- maintenance ---------------------------------------------------------------- export async function clear() { const db = await openAuditDb(); const transaction = db.transaction( - [RUNS_STORE, PREDICTIONS_STORE, RESOLUTIONS_STORE, CASE_BRIEFS_STORE], 'readwrite'); + [RUNS_STORE, PREDICTIONS_STORE, RESOLUTIONS_STORE, CASE_BRIEFS_STORE, CORPUS_EXTRACTS_STORE], 'readwrite'); transaction.objectStore(RUNS_STORE).clear(); transaction.objectStore(PREDICTIONS_STORE).clear(); transaction.objectStore(RESOLUTIONS_STORE).clear(); transaction.objectStore(CASE_BRIEFS_STORE).clear(); + transaction.objectStore(CORPUS_EXTRACTS_STORE).clear(); await tx(transaction); } diff --git a/src/shared/case-synthesis.js b/src/shared/case-synthesis.js index 0eacae8..40f626f 100644 --- a/src/shared/case-synthesis.js +++ b/src/shared/case-synthesis.js @@ -88,6 +88,34 @@ export async function buildMemberUnits(data, { assessmentsByClaim = {} } = {}) { return units; } +/** + * Cache key for ONE member's map-stage extract: a SHA-256 over the exact + * inputs the map call consumes, so a cached extract is reused only when + * re-running would produce the same output. Any change to the sent text, + * the article's claim digest, the case framing, or the prompt version + * yields a new key — which is precisely the invalidation we want (a + * body edit changes `memberText`; a Suggest pass changes `claimsDigest`; + * a prompt bump changes the version). Pure; mirrors the map request the + * runner sends (member_id is derived from the text, so it is omitted). + * + * @param {object} request { memberText, claimsDigest, caseName, scopeQuestion, memberMeta:{title,url} } + * @param {string} [promptVersion] + * @returns {Promise} 64-char hex + */ +export async function corpusExtractKey(request, promptVersion = CORPUS_PROMPT_VERSION) { + const r = request || {}; + const mm = r.memberMeta || {}; + return Crypto.sha256(JSON.stringify({ + v: promptVersion, + text: r.memberText || '', + claims: r.claimsDigest || '', + caseName: r.caseName || '', + scope: r.scopeQuestion || '', + title: mm.title || '', + url: mm.url || '' + })); +} + /** * Content hash over the corpus INPUT — order-insensitive, so it * invalidates a stored brief exactly when membership, member text (the diff --git a/tests/audit-cache.test.mjs b/tests/audit-cache.test.mjs index bc40974..453286b 100644 --- a/tests/audit-cache.test.mjs +++ b/tests/audit-cache.test.mjs @@ -12,7 +12,8 @@ const { saveRun, getRun, runsByArticleHash, listRuns, deleteRun, countRuns, savePrediction, getPrediction, predictionsByArticleHash, predictionsByStatus, saveResolution, getResolution, resolutionsByPredictionCoord, - saveCaseBrief, getCaseBrief, deleteCaseBrief, listCaseBriefs + saveCaseBrief, getCaseBrief, deleteCaseBrief, listCaseBriefs, + saveCorpusExtract, getCorpusExtract, deleteCorpusExtract, listCorpusExtracts, countCorpusExtracts } = await import('../src/shared/audit/audit-cache.js'); const HASH_A = 'a'.repeat(64); @@ -112,3 +113,26 @@ test('case-briefs: CRUD keyed by caseId, coexists with runs (DB v2)', async () = await deleteCaseBrief('case_1'); assert.equal(await getCaseBrief('case_1'), null); }); + +// v3 store — the per-article map-extract cache. +test('corpus-extracts: CRUD keyed by fingerprint, coexists with v1/v2 stores (DB v3)', async () => { + await saveRun({ id: 'audit_y', articleHash: HASH_A }); // v1 + await saveCaseBrief({ caseId: 'case_2', brief: { summary: 'b' } }); // v2 + await saveCorpusExtract({ key: 'k1', extract: { position: { summary: 'p' } }, model: 'm', cachedAt: 100 }); + const got = await getCorpusExtract('k1'); + assert.equal(got.extract.position.summary, 'p'); + assert.equal(got.model, 'm'); + assert.equal(await countCorpusExtracts(), 1); + assert.equal((await getRun('audit_y')).articleHash, HASH_A, 'v1 intact after v3 upgrade'); + assert.equal((await getCaseBrief('case_2')).brief.summary, 'b', 'v2 intact after v3 upgrade'); + // Overwrite by key (same fingerprint = same output = replace in place). + await saveCorpusExtract({ key: 'k1', extract: { position: { summary: 'p2' } } }); + assert.equal((await getCorpusExtract('k1')).extract.position.summary, 'p2'); + assert.equal((await listCorpusExtracts()).length, 1); + await deleteCorpusExtract('k1'); + assert.equal(await getCorpusExtract('k1'), null); + // clear() drops it too. + await saveCorpusExtract({ key: 'k2', extract: {} }); + await clear(); + assert.equal(await countCorpusExtracts(), 0); +}); diff --git a/tests/backup.test.mjs b/tests/backup.test.mjs index ca91fbc..4a8ecec 100644 --- a/tests/backup.test.mjs +++ b/tests/backup.test.mjs @@ -247,8 +247,8 @@ test('estimateBackupSize: withBytes ≥ withoutBytes; counts source docs', async test('dumpDatabase dumps every store of a covered database', async () => { await seedWorkspace(); const dump = await dumpDatabase('xray-audits'); - // 20.4: the case-briefs store (DB v2) is dumped generically too. - assert.deepEqual(Object.keys(dump).sort(), ['case-briefs', 'predictions', 'resolutions', 'runs']); + // 20.4: case-briefs (DB v2) and corpus-extracts (DB v3) are dumped generically too. + assert.deepEqual(Object.keys(dump).sort(), ['case-briefs', 'corpus-extracts', 'predictions', 'resolutions', 'runs']); assert.equal(dump.runs.length, 1); await assert.rejects(() => dumpDatabase('unknown-db'), /no opener/); }); diff --git a/tests/case-synthesis.test.mjs b/tests/case-synthesis.test.mjs index a6b6af7..6025393 100644 --- a/tests/case-synthesis.test.mjs +++ b/tests/case-synthesis.test.mjs @@ -154,6 +154,22 @@ test('case-synthesis: proposalKey is stable and direction-insensitive for relati assert.equal(CS.proposalKey({ kind: 'claim', article_hash: 'A', text: 't' }), 'claim:A|t'); }); +test('case-synthesis: corpusExtractKey is stable on identical inputs, changes on text/claims/prompt', async () => { + const base = { member_id: 'a'.repeat(64), memberText: 'Body text.', claimsDigest: 'c1 — one\nc2 — two', + caseName: 'covid', scopeQuestion: 'origin?', memberMeta: { title: 'T', url: 'https://x/a' } }; + const k = await CS.corpusExtractKey(base); + assert.match(k, /^[0-9a-f]{64}$/); + // Same inputs → same key (deterministic reuse). + assert.equal(await CS.corpusExtractKey({ ...base }), k); + // member_id is derived from text and NOT part of the key. + assert.equal(await CS.corpusExtractKey({ ...base, member_id: 'z'.repeat(64) }), k); + // Each real input flips the key — these are the invalidation triggers. + assert.notEqual(await CS.corpusExtractKey({ ...base, memberText: 'Edited body.' }), k, 'body edit'); + assert.notEqual(await CS.corpusExtractKey({ ...base, claimsDigest: 'c1 — one\nc2 — two\nc3 — three' }), k, 'a Suggest pass added a claim'); + assert.notEqual(await CS.corpusExtractKey(base, 'corpus-v9'), k, 'prompt-version bump'); + assert.notEqual(await CS.corpusExtractKey({ ...base, caseName: 'other' }), k, 'case framing'); +}); + test('case-synthesis: corpusInputHash is order-insensitive but sensitive to membership + prompt', async () => { const a = [{ article_hash: 'h1' }, { article_hash: 'h2' }]; const aRev = [{ article_hash: 'h2' }, { article_hash: 'h1' }];