diff --git a/app/docs/client-mobile-api.md b/app/docs/client-mobile-api.md index adf0b0190..676b5828f 100644 --- a/app/docs/client-mobile-api.md +++ b/app/docs/client-mobile-api.md @@ -60,8 +60,10 @@ shared type set but still return `501`. `chamber`, `type`, `from`, `to`, `order`, and `limit`, and returns the cursor/count/total metadata used by polling clients. - Each feed item's `asset` object carries `name` (the disclosed asset text), - `ticker`, `type`, `sector`, and `marketCapBucket`, plus two enrichment fields - shared with the web client so every surface renders identically: + `ticker`, raw disclosure `type`, `typeName`, canonical cross-chamber + `typeCategory` / `typeCategoryLabel`, `sector`, and `marketCapBucket`, plus + two enrichment fields shared with the web client so every surface renders + identically: - `companyName` — the canonical company name from `securities_ref` (`null` until the ticker is enriched). - `logoUrl` — a same-origin path to the cached logo proxy, e.g. diff --git a/app/docs/fmp-data-sharing.md b/app/docs/fmp-data-sharing.md index e9be71b82..91a45be80 100644 --- a/app/docs/fmp-data-sharing.md +++ b/app/docs/fmp-data-sharing.md @@ -236,7 +236,8 @@ feed rows or public analytics. Per-transaction object (each item in `transactions[]`): ``` -{ id, docId, filerId, txDate, owner, assetName, ticker, assetType, txType, +{ id, docId, filerId, txDate, owner, assetName, ticker, assetType, + assetTypeName, assetTypeCategory, assetTypeCategoryLabel, txType, amountMin, amountMax, isOption, capGainsOver200, rawText, confidence, source, createdAt, cursorSeq, fullName, state, photoUrl, filedDate, firstSeenAt, diff --git a/app/docs/pit-score-export.md b/app/docs/pit-score-export.md index d6c521d85..b8dd0eb95 100644 --- a/app/docs/pit-score-export.md +++ b/app/docs/pit-score-export.md @@ -26,7 +26,10 @@ whether returned rows are safe for historical validation. Rows are keyed by ticker and market-available disclosure timestamp: - `observationId` -- `ticker`, `stableSecurityId`, `cusip`, `cik`, `assetType` +- `ticker`, `stableSecurityId`, `cusip`, `cik` +- raw/security asset context: `assetType`, `assetTypeName` +- cross-chamber canonical asset context: `assetTypeCategory`, + `assetTypeCategoryLabel`, `assetTypeCategorySource` - `tickerMapVersion`, `delistingTickerChangeMetadata` - `asOf`, `disclosureAvailableAt`, `computedAt`, `dataCutoffAt` - `scoreVersion`, `parameterManifest` diff --git a/app/migrations/0019_ingestion_decisions.sql b/app/migrations/0019_ingestion_decisions.sql new file mode 100644 index 000000000..bdeeea19a --- /dev/null +++ b/app/migrations/0019_ingestion_decisions.sql @@ -0,0 +1,25 @@ +-- 0019_ingestion_decisions.sql +-- Append-only audit trail for filing/trade publication decisions. This keeps +-- the review_queue focused on exceptions while preserving every publish/review +-- decision for admin history and future scoring/debugging. + +CREATE TABLE IF NOT EXISTS ingestion_decisions ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + action TEXT NOT NULL, + source TEXT NOT NULL, + actor TEXT, + reason TEXT, + payload TEXT, + transaction_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_doc + ON ingestion_decisions (doc_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_created + ON ingestion_decisions (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_action + ON ingestion_decisions (action, created_at DESC); diff --git a/app/src/admin/__tests__/diagnostics.test.ts b/app/src/admin/__tests__/diagnostics.test.ts index a6f1aa468..d0f59136b 100644 --- a/app/src/admin/__tests__/diagnostics.test.ts +++ b/app/src/admin/__tests__/diagnostics.test.ts @@ -26,6 +26,33 @@ function fakeDb() { ] as T[], }; } + if (/FROM securities_ref/i.test(sql) && /CASE\s+WHEN lower\(source\)/i.test(sql)) { + return { + results: [ + { + provider: 'massive', + calls_total: 2, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24T11:30:00.000Z', + errors_last_24h: 0, + }, + ] as T[], + }; + } + if (/FROM securities_ref/i.test(sql) && /COUNT\(\*\) AS calls_total/i.test(sql)) { + return { + results: [ + { + calls_total: 5, + calls_last_24h: 2, + calls_today: 1, + last_used_at: '2026-06-24T11:00:00.000Z', + errors_last_24h: 1, + }, + ] as T[], + }; + } if (/FROM securities_ref/i.test(sql) && /enrichment_error/i.test(sql)) { return { results: [ @@ -37,6 +64,42 @@ function fakeDb() { ] as T[], }; } + if (/FROM price_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 7, + calls_last_24h: 20, + calls_today: 3, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM spx_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 100, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM tx_performance/i.test(sql)) { + return { + results: [ + { + calls_total: 50, + calls_last_24h: 10, + calls_today: 5, + last_used_at: '2026-06-24T12:30:00.000Z', + }, + ] as T[], + }; + } if (/FROM deliveries/i.test(sql)) return { results: [] as T[] }; if (/FROM review_queue/i.test(sql)) return { results: [] as T[] }; if (/FROM client_commands/i.test(sql)) return { results: [] as T[] }; @@ -72,6 +135,8 @@ describe('admin diagnostics API', () => { ADMIN_TOKEN: 'admin-secret', GEMINI_API_KEY: 'gemini-secret', FMP_API_KEY: 'fmp-secret', + MASSIVE_API_KEY: 'massive-secret', + PRICE_PROVIDER: 'massive', DB: fakeDb(), } as never, ); @@ -91,10 +156,15 @@ describe('admin diagnostics API', () => { callsToday: 1, }), expect.objectContaining({ id: 'source:house', status: 'ok', callsToday: 2 }), + expect.objectContaining({ id: 'provider:massive', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:prices', status: 'ok', configured: true, callsToday: 3 }), + expect.objectContaining({ id: 'cache:spx', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:performance', status: 'ok', configured: true, callsToday: 5 }), ]), ); expect(JSON.stringify(body)).not.toContain('gemini-secret'); expect(JSON.stringify(body)).not.toContain('fmp-secret'); + expect(JSON.stringify(body)).not.toContain('massive-secret'); expect(body.errors).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/app/src/admin/__tests__/reviewQueue.test.ts b/app/src/admin/__tests__/reviewQueue.test.ts index f0f5a5a91..dcff281db 100644 --- a/app/src/admin/__tests__/reviewQueue.test.ts +++ b/app/src/admin/__tests__/reviewQueue.test.ts @@ -43,6 +43,7 @@ describe('review queue admin API', () => { 'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf', raw_object_key: 'raw/H-2026-2003695', doc_kind: 'scanned_pdf', + chamber: 'house', }, ]), } as never, @@ -55,6 +56,7 @@ describe('review queue admin API', () => { sourceUrl: string; rawObjectKey: string; docKind: string; + chamber: string; payload: { minConfidence: number; transactions: unknown[] }; }>; }; @@ -63,6 +65,7 @@ describe('review queue admin API', () => { sourceUrl: 'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf', rawObjectKey: 'raw/H-2026-2003695', docKind: 'scanned_pdf', + chamber: 'house', payload: { minConfidence: 0, transactions: [] }, }); }); @@ -97,6 +100,44 @@ describe('review queue admin API', () => { expect(body.items[0]).toMatchObject({ resolved: true, ingestStatus: 'persisted' }); }); + it('lists ingestion decision history separately from the review queue', async () => { + const res = await app.request( + '/ingestion-decisions', + { headers: { Authorization: 'Bearer admin-secret' } }, + { + ADMIN_TOKEN: 'admin-secret', + DB: fakeDb([ + { + id: 'dec-1', + doc_id: 'S-1', + action: 'auto_published', + source: 'pipeline', + actor: null, + reason: 'passed_normalization', + payload: '{"inserted":2}', + transaction_ids: '["tx1","tx2"]', + created_at: '2026-06-29T00:00:00.000Z', + chamber: 'senate', + ingest_status: 'persisted', + source_url: 'https://example/senate', + }, + ]), + } as never, + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { + available: boolean; + items: Array<{ docId: string; action: string; payload: { inserted: number }; transactionIds: string[] }>; + }; + expect(body.available).toBe(true); + expect(body.items[0]).toMatchObject({ + docId: 'S-1', + action: 'auto_published', + payload: { inserted: 2 }, + transactionIds: ['tx1', 'tx2'], + }); + }); + it('unpublishes a persisted filing: soft-deletes rows, reverts, re-opens review', async () => { // fakeDb whose filing lookup resolves and whose UPDATE reports 3 retracted rows. const db = { @@ -214,12 +255,14 @@ describe('review queue admin API', () => { it("decision='manual' records hand-entered rows as source='manual'", async () => { // Capture the INSERT bind params so we can assert the source column = 'manual'. const binds: unknown[][] = []; + const auditBinds: unknown[][] = []; const db = { prepare(sql: string) { return { _sql: sql, bind(...args: unknown[]) { if (/INSERT OR IGNORE INTO transactions/.test(sql)) binds.push(args); + if (/INSERT INTO ingestion_decisions/.test(sql)) auditBinds.push(args); return this; }, async all() { @@ -256,5 +299,8 @@ describe('review queue admin API', () => { // The transactions INSERT bound source='manual' (it's the 17th positional bind). expect(binds.length).toBe(1); expect(binds[0]).toContain('manual'); + expect(auditBinds.length).toBe(1); + expect(auditBinds[0]).toContain('manual'); + expect(auditBinds[0]).toContain('admin-token'); }); }); diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index 8900d46f6..e30713597 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -29,6 +29,8 @@ import { Hono } from 'hono'; import type { Env, PollConfig, PollWindow, TxType, TxSource } from '../shared/types'; import { all, get, run, type SqlParam } from '../shared/db'; +import { HOUSE_ASSET_TYPE_NAMES } from '../shared/assetTypes'; +import { listIngestionDecisions, recordIngestionDecision } from '../shared/ingestionDecisions'; import { getConfig, setConfig } from '../shared/config'; import { uuid } from '../shared/ids'; import { listSubscriptions } from '../delivery/subscriptions'; @@ -111,6 +113,14 @@ function isExplicitOpenAdmin(env: EnvWithAdmin): boolean { return env.ADMIN_OPEN_IN_DEV === 'true'; } +function adminActor(c: { req: { header(name: string): string | undefined } }): string { + const accessEmail = + c.req.header('Cf-Access-Authenticated-User-Email') || + c.req.header('cf-access-authenticated-user-email'); + if (accessEmail) return accessEmail; + return c.req.header('authorization') ? 'admin-token' : 'admin'; +} + /** * Admin auth — authorized if a valid bearer token OR an allowlisted, verified * Cloudflare Access identity is presented. Open only when neither is configured. @@ -209,6 +219,7 @@ interface ReviewRow { source_url: string | null; raw_object_key: string | null; doc_kind: string | null; + chamber?: string | null; } interface DiagnosticConnection { @@ -268,6 +279,7 @@ interface EditedTx { assetName?: string; ticker?: string | null; assetType?: string | null; + assetTypeName?: string | null; txType?: TxType; amountMin?: number | null; amountMax?: number | null; @@ -277,6 +289,15 @@ interface EditedTx { confidence?: number; } +function reviewAssetTypeName(e: EditedTx): string | null { + const supplied = typeof e.assetTypeName === 'string' ? e.assetTypeName.trim() : ''; + if (supplied) return supplied; + const raw = typeof e.assetType === 'string' ? e.assetType.trim() : ''; + if (!raw || raw.toLowerCase() === 'unknown') return null; + const code = raw.toUpperCase(); + return HOUSE_ASSET_TYPE_NAMES[code] ?? raw; +} + // --- Member photo enrichment (name -> bioguide -> unitedstates/images CDN) --- const LEGISLATOR_SOURCES = [ @@ -514,6 +535,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { f.source_url, f.raw_object_key, f.doc_kind, + f.chamber, f.ingest_status, (SELECT COUNT(*) FROM transactions t WHERE t.doc_id = rq.doc_id AND t.source = 'manual' AND t.deprecated_at IS NULL) AS manual_rows, @@ -595,12 +617,32 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { sourceUrl: row.source_url ?? '', rawObjectKey: row.raw_object_key ?? '', docKind: row.doc_kind ?? '', + chamber: row.chamber ?? '', models: modelsByDoc.get(row.doc_id) ?? [], }; }); return c.json({ items, count: items.length, resolved: resolved === 1 }); }); + // --- GET /ingestion-decisions ------------------------------------------ + // Append-only filing/trade decision history. Unlike review_queue, this also + // includes clean auto-published filings that never needed human review. + r.get('/ingestion-decisions', async (c) => { + const rawLimit = parseInt(c.req.query('limit') || '100', 10); + const limit = Number.isFinite(rawLimit) ? rawLimit : 100; + const docId = c.req.query('docId') || null; + try { + const items = await listIngestionDecisions(c.env.DB, { limit, docId }); + return c.json({ items, count: items.length, available: true }); + } catch (err) { + const msg = (err as Error).message; + if (/no such table|ingestion_decisions/i.test(msg)) { + return c.json({ items: [], count: 0, available: false }); + } + return c.json({ error: msg }, 500); + } + }); + // --- GET /review/:docId/extractions ------------------------------------- // Full stored readings (result_json) for one document, newest first — powers // the dashboard's "view each model's reading" panel. Separate from the list @@ -681,11 +723,24 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { } if (decision === 'reject') { + const nowIso = new Date().toISOString(); await run(c.env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [docId]); await run(c.env.DB, 'UPDATE filings SET ingest_status = ? WHERE doc_id = ?', [ 'error', docId, ]); + await recordIngestionDecision(c.env.DB, { + docId, + action: 'rejected', + source: 'admin', + actor: adminActor(c), + reason: review.reason ?? 'rejected', + payload: { + reviewCreatedAt: review.created_at, + reviewPayload: review.payload ? safeJson(review.payload) : null, + }, + createdAt: nowIso, + }); return c.json({ docId, decision: 'reject', resolved: true }); } @@ -726,6 +781,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const nowIso = new Date().toISOString(); for (const [rowIndex, e] of edits.entries()) { const id = uuid(); + const assetTypeName = reviewAssetTypeName(e); const rowKey = transactionRowKey(source, rowIndex, { txDate: e.txDate ?? null, owner: e.owner === 'self' || e.owner === 'spouse' || e.owner === 'joint' || e.owner === 'dependent' @@ -734,6 +790,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { assetName: e.assetName ?? '', ticker: e.ticker ?? null, assetType: e.assetType ?? null, + assetTypeName, txType: (e.txType as TxType) ?? 'P', amountMin: e.amountMin ?? null, amountMax: e.amountMax ?? null, @@ -747,8 +804,8 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { `INSERT OR IGNORE INTO transactions ( id, doc_id, filer_id, tx_date, owner, asset_name, ticker, asset_type, tx_type, amount_min, amount_max, is_option, cap_gains_over_200, - raw_text, row_key, confidence, source, created_at, cursor_seq - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`, + raw_text, asset_type_name, row_key, confidence, source, created_at, cursor_seq + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`, [ id, docId, @@ -764,6 +821,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { e.isOption ? 1 : 0, e.capGainsOver200 ? 1 : 0, e.rawText ?? '', + assetTypeName, rowKey, e.confidence ?? 1, source, @@ -789,6 +847,22 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { } } + await recordIngestionDecision(c.env.DB, { + docId, + action: decision === 'manual' ? 'manual' : 'confirmed', + source: 'admin', + actor: adminActor(c), + reason: review.reason ?? null, + transactionIds: insertedIds, + payload: { + source, + editCount: edits.length, + inserted: insertedIds.length, + reviewCreatedAt: review.created_at, + }, + createdAt: nowIso, + }); + return c.json({ docId, decision, @@ -846,6 +920,16 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { [docId, 'unpublished: ' + reason, null, nowIso], ); + await recordIngestionDecision(c.env.DB, { + docId, + action: 'unpublished', + source: 'admin', + actor: adminActor(c), + reason, + payload: { deprecatedTransactions: deprecated }, + createdAt: nowIso, + }); + return c.json({ docId, unpublished: true, deprecatedTransactions: deprecated, reason }); }); @@ -897,6 +981,12 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const env = c.env as Env & { GEMINI_API_KEY?: string; FMP_API_KEY?: string; + MASSIVE_API_KEY?: string; + INTRINIO_API_KEY?: string; + TWELVEDATA_API_KEY?: string; + FINNHUB_API_KEY?: string; + LOGODEV_PUBLISHABLE_KEY?: string; + PRICE_PROVIDER?: string; WEBHOOK_SIGNING_KEY?: string; GOOGLE_OAUTH_CLIENT_ID?: string; RESEND_API_KEY?: string; @@ -990,7 +1080,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const fmpRow = fmp[0]; connections.push({ id: 'provider:fmp', - label: 'FMP Market Data', + label: 'FMP Enrichment', status: connectionStatus(!!env.FMP_API_KEY, fmpRow?.errors_last_24h ?? 0, fmpRow?.last_used_at ?? null), configured: !!env.FMP_API_KEY, lastUsedAt: fmpRow?.last_used_at ?? null, @@ -1001,6 +1091,142 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { note: env.FMP_API_KEY ? 'Enrichment rows refreshed' : 'FMP_API_KEY is not configured', }); + const providerRows = await optionalAll<{ + provider: string; + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + errors_last_24h: number; + }>( + c.env, + `SELECT CASE + WHEN lower(source) LIKE '%massive%' THEN 'massive' + WHEN lower(source) LIKE '%intrinio%' THEN 'intrinio' + WHEN lower(source) LIKE '%twelvedata%' THEN 'twelvedata' + WHEN lower(source) LIKE '%finnhub%' THEN 'finnhub' + WHEN lower(source) LIKE '%edgar%' THEN 'edgar' + ELSE 'other' + END AS provider, + COUNT(*) AS calls_total, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(enriched_at) AS last_used_at, + SUM(CASE WHEN enrichment_error IS NOT NULL AND enrichment_error != '' AND enriched_at >= ? THEN 1 ELSE 0 END) AS errors_last_24h + FROM securities_ref + WHERE source IS NOT NULL AND source != '' + GROUP BY provider`, + [last24, today, last24], + ); + const providerUsage = new Map(providerRows.map((row) => [row.provider, row])); + const addMarketProvider = (id: string, label: string, configured: boolean, note: string) => { + const row = providerUsage.get(id); + connections.push({ + id: `provider:${id}`, + label, + status: connectionStatus(configured, row?.errors_last_24h ?? 0, row?.last_used_at ?? null), + configured, + lastUsedAt: row?.last_used_at ?? null, + callsTotal: row?.calls_total ?? 0, + callsLast24h: row?.calls_last_24h ?? 0, + callsToday: row?.calls_today ?? 0, + errorsLast24h: row?.errors_last_24h ?? 0, + note, + }); + }; + addMarketProvider('massive', 'Massive Market Data', !!env.MASSIVE_API_KEY, env.MASSIVE_API_KEY ? 'Reference/price fallback configured' : 'MASSIVE_API_KEY is not configured'); + addMarketProvider('intrinio', 'Intrinio Reference Data', !!env.INTRINIO_API_KEY, env.INTRINIO_API_KEY ? 'Reference fallback configured' : 'INTRINIO_API_KEY is not configured'); + addMarketProvider('twelvedata', 'Twelve Data Reference', !!env.TWELVEDATA_API_KEY, env.TWELVEDATA_API_KEY ? 'Reference fallback configured' : 'TWELVEDATA_API_KEY is not configured'); + addMarketProvider('finnhub', 'Finnhub Reference', !!env.FINNHUB_API_KEY, env.FINNHUB_API_KEY ? 'Reference fallback configured' : 'FINNHUB_API_KEY is not configured'); + addMarketProvider('edgar', 'SEC EDGAR Reference', true, 'Free fallback; no secret required'); + + const priceRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(DISTINCT ticker) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM price_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const priceRow = priceRows[0]; + const hasPriceProvider = !!(env.FMP_API_KEY || env.MASSIVE_API_KEY); + connections.push({ + id: 'cache:prices', + label: 'Asset Price Cache', + status: connectionStatus(hasPriceProvider, 0, priceRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: priceRow?.last_used_at ?? null, + callsTotal: priceRow?.calls_total ?? 0, + callsLast24h: priceRow?.calls_last_24h ?? 0, + callsToday: priceRow?.calls_today ?? 0, + errorsLast24h: 0, + note: hasPriceProvider + ? `PRICE_PROVIDER=${env.PRICE_PROVIDER || 'fmp'}; counts show cached assets/rows, not raw API calls` + : 'No FMP_API_KEY or MASSIVE_API_KEY configured for price history', + }); + + const spxRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM spx_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const spxRow = spxRows[0]; + connections.push({ + id: 'cache:spx', + label: 'S&P Benchmark Cache', + status: connectionStatus(hasPriceProvider, 0, spxRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: spxRow?.last_used_at ?? null, + callsTotal: spxRow?.calls_total ?? 0, + callsLast24h: spxRow?.calls_last_24h ?? 0, + callsToday: spxRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'SPY-adjusted close history used as the S&P comparison baseline', + }); + + const perfRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(computed_at) AS last_used_at + FROM tx_performance`, + [last24, today], + ); + const perfRow = perfRows[0]; + connections.push({ + id: 'cache:performance', + label: 'Trade Performance Anchors', + status: connectionStatus(hasPriceProvider, 0, perfRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: perfRow?.last_used_at ?? null, + callsTotal: perfRow?.calls_total ?? 0, + callsLast24h: perfRow?.calls_last_24h ?? 0, + callsToday: perfRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'Required for per-trade and member S&P-relative performance', + }); + const webhooks = await optionalAll<{ calls_total: number; calls_last_24h: number; @@ -1971,6 +2197,21 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { // 0018_agreement_attempted.sql — one autonomous agreement attempt per review doc. 'ALTER TABLE review_queue ADD COLUMN agreement_attempted_at TEXT', 'CREATE INDEX IF NOT EXISTS idx_review_queue_agreement ON review_queue (agreement_attempted_at)', + // 0019_ingestion_decisions.sql — append-only audit trail for publication/review decisions. + `CREATE TABLE IF NOT EXISTS ingestion_decisions ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + action TEXT NOT NULL, + source TEXT NOT NULL, + actor TEXT, + reason TEXT, + payload TEXT, + transaction_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL + )`, + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_doc ON ingestion_decisions (doc_id, created_at DESC)', + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_created ON ingestion_decisions (created_at DESC)', + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_action ON ingestion_decisions (action, created_at DESC)', ]; const applied: string[] = []; const skipped: string[] = []; diff --git a/app/src/analytics/__tests__/builders.test.ts b/app/src/analytics/__tests__/builders.test.ts index 21f2ed996..4e7fdec2c 100644 --- a/app/src/analytics/__tests__/builders.test.ts +++ b/app/src/analytics/__tests__/builders.test.ts @@ -173,10 +173,12 @@ describe('buildPartySplitQuery', () => { }); describe('buildSectorBreakdownQuery', () => { - it('buckets the asset_type and coalesces empties to Unknown', () => { + it('buckets canonical asset-type categories across raw source labels', () => { const q = buildSectorBreakdownQuery({ window: 'all' }); - expect(q.sql).toContain("COALESCE(NULLIF(t.asset_type, ''), 'Unknown') AS asset_type"); - expect(q.sql).toContain('GROUP BY asset_type'); + expect(q.sql).toContain('AS asset_type_category'); + expect(q.sql).toContain("THEN 'public_equity'"); + expect(q.sql).toContain("THEN 'fixed_income_government'"); + expect(q.sql).toContain('GROUP BY asset_type_category'); }); }); diff --git a/app/src/analytics/builders.ts b/app/src/analytics/builders.ts index 4b0b2b4f8..399718f06 100644 --- a/app/src/analytics/builders.ts +++ b/app/src/analytics/builders.ts @@ -15,6 +15,7 @@ */ import type { SqlParam } from '../shared/db'; +import { canonicalAssetTypeCategorySql } from '../shared/assetTypes'; import { ANALYTICS_FROM_JOINS, ANALYTICS_FROM_JOINS_SECURITIES, @@ -321,21 +322,23 @@ export function buildPartySplitOverTimeQuery( } // --------------------------------------------------------------------------- -// 8. Sector breakdown — by asset_type (the only classification we have) +// 8. Instrument-type breakdown - canonicalized across House codes and Senate labels. // --------------------------------------------------------------------------- export function buildSectorBreakdownQuery(p: CommonFilters & { limit?: number }): BuiltQuery { const { where, params } = buildCommonFilters(p); const limit = clampLimit(p.limit, 20, 100); + const categorySql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); const sql = - "SELECT COALESCE(NULLIF(t.asset_type, ''), 'Unknown') AS asset_type, " + + `SELECT ${categorySql} AS asset_type_category, ` + + "GROUP_CONCAT(DISTINCT COALESCE(NULLIF(t.asset_type, ''), 'Unknown')) AS raw_asset_types, " + 'COUNT(*) AS trade_count, ' + `${BUY} AS buy_count, ${SELL} AS sell_count, ` + `SUM(${MID}) AS est_volume, ` + `COUNT(DISTINCT CASE WHEN ${TICKER_RESOLVED_SQL} THEN t.ticker END) AS unique_tickers ` + ANALYTICS_FROM_JOINS + whereSql(where) + - 'GROUP BY asset_type ORDER BY trade_count DESC ' + + 'GROUP BY asset_type_category ORDER BY trade_count DESC ' + `LIMIT ${limit}`; return { sql, params }; } diff --git a/app/src/analytics/routes.ts b/app/src/analytics/routes.ts index a461f26ba..62565a19b 100644 --- a/app/src/analytics/routes.ts +++ b/app/src/analytics/routes.ts @@ -20,6 +20,7 @@ import { Hono } from 'hono'; import type { Env } from '../shared/types'; import { all, get, parseJson } from '../shared/db'; +import { assetTypeCategoryLabel, isAssetTypeCategory } from '../shared/assetTypes'; import { asChamber, asPartyBucket, @@ -95,6 +96,14 @@ function usd(v: unknown): number { function str(v: unknown): string | null { return typeof v === 'string' && v.length > 0 ? v : null; } +function assetTypeCategory(v: unknown) { + const s = str(v); + return isAssetTypeCategory(s) ? s : 'unknown'; +} +function rawList(v: unknown): string[] { + const s = str(v); + return s ? s.split(',').map((part) => part.trim()).filter(Boolean) : []; +} /** Split a list into fixed-size batches (to stay under D1's 100-bound-param cap). */ function chunk(items: T[], size: number): T[][] { const out: T[][] = []; @@ -658,7 +667,9 @@ export function buildAnalyticsRouter(): Hono<{ Bindings: Env }> { const built = buildSectorBreakdownQuery({ ...f, limit }); const rows = await all>(c.env.DB, built.sql, built.params); const sectors = rows.map((row) => ({ - assetType: str(row.asset_type) ?? 'Unknown', + assetType: assetTypeCategoryLabel(assetTypeCategory(row.asset_type_category)), + assetTypeCategory: assetTypeCategory(row.asset_type_category), + rawAssetTypes: rawList(row.raw_asset_types), tradeCount: num(row.trade_count), buyCount: num(row.buy_count), sellCount: num(row.sell_count), diff --git a/app/src/client/__tests__/routes.test.ts b/app/src/client/__tests__/routes.test.ts index ea0522a96..8edcaf6b4 100644 --- a/app/src/client/__tests__/routes.test.ts +++ b/app/src/client/__tests__/routes.test.ts @@ -288,7 +288,14 @@ describe('client API routes', () => { expect(res.status).toBe(200); const body = (await res.json()) as { items: Array<{ - asset: { ticker: string | null; companyName: string | null; logoUrl: string | null; sector: string | null }; + asset: { + ticker: string | null; + companyName: string | null; + logoUrl: string | null; + sector: string | null; + typeCategory: string; + typeCategoryLabel: string; + }; }>; }; expect(body.items[0].asset).toMatchObject({ @@ -296,6 +303,8 @@ describe('client API routes', () => { companyName: 'Apple Inc.', logoUrl: '/api/logos/ticker?symbol=AAPL', sector: 'Technology', + typeCategory: 'public_equity', + typeCategoryLabel: 'Public Equity', }); }); @@ -336,9 +345,23 @@ describe('client API routes', () => { const res = await app.request('http://localhost/feed?limit=1', {}, env); expect(res.status).toBe(200); const body = (await res.json()) as { - items: Array<{ asset: { ticker: string | null; companyName: string | null; logoUrl: string | null } }>; + items: Array<{ + asset: { + ticker: string | null; + companyName: string | null; + logoUrl: string | null; + typeCategory: string; + typeCategoryLabel: string; + }; + }>; }; - expect(body.items[0].asset).toMatchObject({ ticker: null, companyName: null, logoUrl: null }); + expect(body.items[0].asset).toMatchObject({ + ticker: null, + companyName: null, + logoUrl: null, + typeCategory: 'fixed_income_government', + typeCategoryLabel: 'Government / Municipal Debt', + }); }); it('updates preferences through an authenticated command', async () => { diff --git a/app/src/client/routes.ts b/app/src/client/routes.ts index 69fe6ee9b..11a2efdcf 100644 --- a/app/src/client/routes.ts +++ b/app/src/client/routes.ts @@ -28,6 +28,7 @@ import { type FeedTransactionRow, type TxQueryParams, } from '../delivery/rows'; +import { canonicalizeAssetType } from '../shared/assetTypes'; import { createSubscription, getSubscription, @@ -203,6 +204,10 @@ function clientLogoUrl(ticker: string | null): string | null { function clientTradeFromRow(row: FeedTransactionRow & { __chamber?: string | null; __member_name?: string | null; __party?: string | null }): ClientTrade { const tx = mapFeedTransaction(row); + const assetType = canonicalizeAssetType(tx.assetType, tx.assetTypeName, { + isOption: tx.isOption, + assetName: tx.assetName, + }); return { id: tx.id, cursor: tx.cursorSeq, @@ -221,6 +226,9 @@ function clientTradeFromRow(row: FeedTransactionRow & { __chamber?: string | nul companyName: tx.refCompanyName ?? null, logoUrl: clientLogoUrl(tx.ticker), type: tx.assetType, + typeName: tx.assetTypeName ?? null, + typeCategory: assetType.category, + typeCategoryLabel: assetType.categoryLabel, sector: tx.refSector ?? null, marketCapBucket: tx.refMarketCapBucket ?? null, }, diff --git a/app/src/delivery/rows.ts b/app/src/delivery/rows.ts index 40e7d7670..15b8e00df 100644 --- a/app/src/delivery/rows.ts +++ b/app/src/delivery/rows.ts @@ -20,6 +20,7 @@ import type { TxType, } from '../shared/types'; import { parseJson, toBool } from '../shared/db'; +import { canonicalizeAssetType } from '../shared/assetTypes'; // --------------------------------------------------------------------------- // Raw row shapes (mirror the D1 column names in migrations/0001_init.sql) @@ -112,6 +113,11 @@ export interface FilingRow { // --------------------------------------------------------------------------- export function mapTransaction(row: TransactionRow): Transaction { + const isOption = toBool(row.is_option); + const assetType = canonicalizeAssetType(row.asset_type, row.asset_type_name ?? null, { + isOption, + assetName: row.asset_name ?? null, + }); return { id: row.id, docId: row.doc_id, @@ -122,10 +128,12 @@ export function mapTransaction(row: TransactionRow): Transaction { ticker: row.ticker, assetType: row.asset_type, assetTypeName: row.asset_type_name ?? null, + assetTypeCategory: assetType.category, + assetTypeCategoryLabel: assetType.categoryLabel, txType: (row.tx_type as TxType) ?? 'P', amountMin: row.amount_min, amountMax: row.amount_max, - isOption: toBool(row.is_option), + isOption, capGainsOver200: toBool(row.cap_gains_over_200), rawText: row.raw_text ?? '', filingStatus: row.filing_status ?? null, diff --git a/app/src/export/__tests__/pitScores.test.ts b/app/src/export/__tests__/pitScores.test.ts index 2d3a5b3cb..79d1f2e14 100644 --- a/app/src/export/__tests__/pitScores.test.ts +++ b/app/src/export/__tests__/pitScores.test.ts @@ -144,6 +144,16 @@ describe('buildPitScoreExport pagination', () => { const first = await buildPitScoreExport(env as never, { limit: 1, format: 'json', placebo: 'none', source: 'all' }, new Date('2026-03-01T00:00:00.000Z')); expect(first.rows.map((r) => r.ticker)).toEqual(['AAPL']); expect(first.pagination.nextCursor).toBe('2026-01-01T00:00:00.000Z~AAPL'); + expect(first.rows[0]).toMatchObject({ + assetTypeCategory: 'public_equity', + assetTypeCategoryLabel: 'Public Equity', + assetTypeCategorySource: 'label', + }); + expect(first.rows[0].includedDisclosures[0]).toMatchObject({ + assetType: 'STOCK', + assetTypeName: 'Stock', + assetTypeCategory: 'public_equity', + }); expect(first.validationReadiness).toMatchObject({ historicalValidationReady: false, scoreInputsPitSafeRows: 1, @@ -169,6 +179,33 @@ describe('buildPitScoreExport pagination', () => { expect(second.pagination.nextCursor).toBeNull(); }); + it('uses securities_ref asset class when disclosure asset type is missing', async () => { + const env = { + DB: fakeDb({ + transactions: [ + tx('tx1', 'AAPL', '2026-01-01T00:00:00.000Z', { + asset_type: null, + asset_type_name: null, + asset_class: 'equity', + }), + ], + price_eod: [], + spx_eod: [], + }), + }; + const result = await buildPitScoreExport( + env as never, + { limit: 1, format: 'json', placebo: 'none', source: 'all' }, + new Date('2026-03-01T00:00:00.000Z'), + ); + + expect(result.rows[0]).toMatchObject({ + assetType: 'equity', + assetTypeCategory: 'public_equity', + assetTypeCategorySource: 'label', + }); + }); + it('flags seed/date-only rows as not true historical validation rows', async () => { const env = { DB: fakeDb({ diff --git a/app/src/export/pitScores.ts b/app/src/export/pitScores.ts index e45e51bf4..39fdaeb0a 100644 --- a/app/src/export/pitScores.ts +++ b/app/src/export/pitScores.ts @@ -14,6 +14,7 @@ import { bracketMidpoint, netSentiment, round } from '../analytics/compute'; import { committeeConflict } from '../analytics/conflicts'; import { TICKER_ALIASES } from '../extraction/tickerNormalize'; import { pctChange } from '../prices/compute'; +import { canonicalizeAssetType } from '../shared/assetTypes'; export const PIT_SCORE_VERSION = 'congress-pit-v2'; export const TICKER_MAP_VERSION = 'ticker-normalize-v1'; @@ -158,6 +159,10 @@ interface PitScoreRow { cusip: string | null; cik: string | null; assetType: string | null; + assetTypeName: string | null; + assetTypeCategory: string; + assetTypeCategoryLabel: string; + assetTypeCategorySource: string; tickerMapVersion: string; delistingTickerChangeMetadata: Record; asOf: string; @@ -1197,25 +1202,45 @@ async function buildRow( const signedScore = score == null ? null : direction === 'SELL' ? -score : direction === 'BUY' ? score : 0; const price = await (priceCache.get(ticker) ?? priceCache.set(ticker, priceSeries(env, ticker)).get(ticker)!); const ref = txs.find((t) => t.company_name || t.cik || t.asset_class || t.sector) ?? txs[0]; - const includedDisclosures = txs.map((t) => ({ - availabilitySource: availabilityFor(t).source, - availabilityPrecision: availabilityFor(t).precision, - disclosureId: t.id, - docId: t.doc_id, - sourceUrl: t.source_url, - hashedFilerId: stableFilerHash(t.filer_id), - txDate: t.tx_date, - disclosedAt: isoTimestamp(t.first_seen_at) ?? isoTimestamp(t.filed_date) ?? t.created_at, - filedAt: t.filed_date, - side: t.tx_type, - owner: t.owner, - amountLow: finiteOrNull(t.amount_min), - amountHigh: finiteOrNull(t.amount_max), - amountEstimate: Math.round(midpoint(t)), - chamber: t.filer_chamber ?? t.filing_chamber, - amendmentFlag: false, - cancelFlag: false, - })); + const disclosureAssetType = canonicalizeAssetType(ref?.asset_type ?? null, ref?.asset_type_name ?? null, { + isOption: txs.some((t) => t.is_option === 1), + assetName: ref?.asset_name ?? null, + }); + const canonicalAssetType = disclosureAssetType.category === 'unknown' && ref?.asset_class + ? canonicalizeAssetType(ref.asset_class, ref?.asset_type_name ?? null, { + isOption: txs.some((t) => t.is_option === 1), + assetName: ref?.asset_name ?? null, + }) + : disclosureAssetType; + const includedDisclosures = txs.map((t) => { + const disclosureAssetType = canonicalizeAssetType(t.asset_type, t.asset_type_name, { + isOption: t.is_option === 1, + assetName: t.asset_name, + }); + return { + availabilitySource: availabilityFor(t).source, + availabilityPrecision: availabilityFor(t).precision, + disclosureId: t.id, + docId: t.doc_id, + sourceUrl: t.source_url, + hashedFilerId: stableFilerHash(t.filer_id), + txDate: t.tx_date, + disclosedAt: isoTimestamp(t.first_seen_at) ?? isoTimestamp(t.filed_date) ?? t.created_at, + filedAt: t.filed_date, + side: t.tx_type, + owner: t.owner, + amountLow: finiteOrNull(t.amount_min), + amountHigh: finiteOrNull(t.amount_max), + amountEstimate: Math.round(midpoint(t)), + chamber: t.filer_chamber ?? t.filing_chamber, + assetType: t.asset_type, + assetTypeName: t.asset_type_name, + assetTypeCategory: disclosureAssetType.category, + assetTypeCategoryLabel: disclosureAssetType.categoryLabel, + amendmentFlag: false, + cancelFlag: false, + }; + }); const aliasesFrom = Object.entries(TICKER_ALIASES).filter(([, to]) => to === ticker).map(([from]) => from); const clusterConsensus = buildClusterConsensus(ticker, asOf, txs, allTxRows, memberSkill); const pitValidity = buildPitValidity(txs); @@ -1226,6 +1251,10 @@ async function buildRow( cusip: null, cik: ref?.cik ?? null, assetType: ref?.asset_class ?? ref?.asset_type ?? null, + assetTypeName: ref?.asset_type_name ?? null, + assetTypeCategory: canonicalAssetType.category, + assetTypeCategoryLabel: canonicalAssetType.categoryLabel, + assetTypeCategorySource: canonicalAssetType.source, tickerMapVersion: TICKER_MAP_VERSION, delistingTickerChangeMetadata: { knownPriorTickers: aliasesFrom, diff --git a/app/src/extraction/__tests__/normalizer.test.ts b/app/src/extraction/__tests__/normalizer.test.ts index 88fb7add5..2279e51b8 100644 --- a/app/src/extraction/__tests__/normalizer.test.ts +++ b/app/src/extraction/__tests__/normalizer.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { normalize, CONFIDENCE_THRESHOLD } from '../normalizer'; +import { normalize, CONFIDENCE_THRESHOLD, transactionRowKey } from '../normalizer'; import type { Env, Filing, ParsedTx } from '../../shared/types'; // --------------------------------------------------------------------------- @@ -101,6 +101,17 @@ const tx = (over: Partial = {}): ParsedTx => ({ }); describe('normalize', () => { + it('keeps row identity stable when asset type labels are added as enrichment', () => { + const parsed = tx({ assetType: 'Stock', assetTypeName: null }); + const enriched = tx({ assetType: 'Stock', assetTypeName: 'Stock' }); + + expect(transactionRowKey('primary', 0, enriched)).toBe(transactionRowKey('primary', 0, parsed)); + + const house = tx({ assetType: 'ST', assetTypeName: 'Stocks (including ADRs)' }); + const houseWithoutLabel = tx({ assetType: 'ST', assetTypeName: null }); + expect(transactionRowKey('primary', 0, house)).not.toBe(transactionRowKey('primary', 0, houseWithoutLabel)); + }); + it('publishes high-confidence, resolved, valid rows', async () => { const { env, cap } = makeEnv([{ ticker: 'AAPL', name: 'Apple Inc.', aliases: '["Apple"]' }]); const result = await normalize(env, filing(), [tx()]); diff --git a/app/src/extraction/__tests__/senateHtml.test.ts b/app/src/extraction/__tests__/senateHtml.test.ts index ab718e235..90eb76a73 100644 --- a/app/src/extraction/__tests__/senateHtml.test.ts +++ b/app/src/extraction/__tests__/senateHtml.test.ts @@ -75,6 +75,8 @@ describe('SenateHtmlExtractor', () => { expect(r1.owner).toBe('spouse'); expect(r1.ticker).toBe('AAPL'); expect(r1.assetName).toContain('Apple'); + expect(r1.assetType).toBe('Stock'); + expect(r1.assetTypeName).toBe('Stock'); expect(r1.txType).toBe('P'); expect(r1.amountMin).toBe(1001); expect(r1.amountMax).toBe(15000); diff --git a/app/src/extraction/agreement.ts b/app/src/extraction/agreement.ts index 5df4397d8..613bbebab 100644 --- a/app/src/extraction/agreement.ts +++ b/app/src/extraction/agreement.ts @@ -16,6 +16,7 @@ import { runCandidateOnDoc, type BakeoffCandidate, type CandidateDocResult } fro import { arbitrationRowKey } from '../extractors/types'; import { recomputeTransactions, persistTransactions, HARD_FAILURE_FLAGS } from './normalizer'; import { mapFiling, type FilingRow } from '../delivery/rows'; +import { recordIngestionDecision } from '../shared/ingestionDecisions'; export interface AgreementModels { a: BakeoffCandidate; @@ -102,6 +103,24 @@ export async function processAgreementDoc( const insertedIds = await persistTransactions(env, txs); await run(env.DB, "UPDATE filings SET ingest_status = 'persisted', error = NULL WHERE doc_id = ?", [docId]); await run(env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [docId]); + if (insertedIds.length > 0) { + await recordIngestionDecision(env.DB, { + docId, + action: 'agreement_published', + source: 'agreement', + reason: 'model_agreement', + transactionIds: insertedIds, + payload: { + rowCount: flagged.length, + inserted: insertedIds.length, + models: { + a: `${models.a.provider}:${models.a.model}`, + b: `${models.b.provider}:${models.b.model}`, + ...(models.c ? { c: `${models.c.provider}:${models.c.model}` } : {}), + }, + }, + }); + } for (const txId of insertedIds) { try { await env.DELIVERY_QUEUE.send({ type: 'delivery.dispatch', txId }); } catch { /* best-effort */ } } diff --git a/app/src/extraction/normalizer.ts b/app/src/extraction/normalizer.ts index 920ed71ed..31cb5c752 100644 --- a/app/src/extraction/normalizer.ts +++ b/app/src/extraction/normalizer.ts @@ -21,7 +21,9 @@ import type { Env, Filing, Owner, ParsedTx, Transaction, TxType } from '../shared/types'; import { all, run, fromBool, parseJson } from '../shared/db'; import { isValidBracket, matchBracket, nearestBracket } from '../shared/brackets'; +import { canonicalizeAssetType } from '../shared/assetTypes'; import { uuid } from '../shared/ids'; +import { recordIngestionDecision } from '../shared/ingestionDecisions'; import { isPlaceholderTicker, resolveTickerDeterministic, TICKER_ALIASES } from './tickerNormalize'; /** @@ -94,7 +96,7 @@ export function transactionRowKey( normalizeText(fields.assetName), (fields.ticker ?? '').toUpperCase(), normalizeText(fields.assetType), - normalizeText(fields.assetTypeName ?? null), + rowKeyAssetTypeName(fields.assetType, fields.assetTypeName ?? null), fields.txType ?? '', fields.amountMin ?? '', fields.amountMax ?? '', @@ -110,6 +112,12 @@ export function transactionRowKey( return `v1:${source}:${rowIndex}:${fnv1a32(payload)}`; } +function rowKeyAssetTypeName(assetType: string | null | undefined, assetTypeName: string | null): string { + const type = normalizeText(assetType ?? null); + const name = normalizeText(assetTypeName); + return name && name !== type ? name : ''; +} + /** * Re-derive Transaction rows (with the current, recalibrated confidence rubric) * from parsed rows WITHOUT any DB write or delivery fan-out. Shared by normalize() @@ -185,10 +193,24 @@ export async function normalize( ); if (needsReview) { + const reason = reviewReason(flagged, minConfidence); await routeToReview(env, filing, flagged, minConfidence, nowIso, { extractor: extractorName, modelVersion, }); + await recordIngestionDecision(env.DB, { + docId: filing.docId, + action: 'review_opened', + source: 'pipeline', + reason, + payload: { + minConfidence, + extractor: extractorName, + modelVersion, + transactionCount: transactions.length, + }, + createdAt: nowIso, + }); return { transactions, minConfidence, needsReview: true }; } @@ -198,6 +220,23 @@ export async function normalize( await run(env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [filing.docId]); const insertedIds = await persistTransactions(env, transactions); + if (insertedIds.length > 0) { + await recordIngestionDecision(env.DB, { + docId: filing.docId, + action: 'auto_published', + source: 'pipeline', + reason: 'passed_normalization', + transactionIds: insertedIds, + payload: { + minConfidence, + extractor: extractorName, + modelVersion, + transactionCount: transactions.length, + inserted: insertedIds.length, + }, + createdAt: nowIso, + }); + } // Fan out delivery only for rows D1 actually inserted. Retries/concurrent // normalizations hit the unique row key and are ignored without duplicate @@ -238,6 +277,10 @@ function buildTransaction( filing.filedDate, resolve, ); + const assetType = canonicalizeAssetType(p.assetType, p.assetTypeName ?? null, { + isOption: p.isOption, + assetName: p.assetName, + }); const tx: Transaction = { id: uuid(), @@ -249,6 +292,8 @@ function buildTransaction( ticker: s.ticker, assetType: p.assetType, assetTypeName: p.assetTypeName ?? null, + assetTypeCategory: assetType.category, + assetTypeCategoryLabel: assetType.categoryLabel, txType: s.txType, amountMin: s.amountMin, amountMax: s.amountMax, @@ -499,12 +544,7 @@ async function routeToReview( nowIso: string, meta: { extractor: string | null; modelVersion: string | null }, ): Promise { - const reasons = new Set(); - for (const f of flagged) for (const flag of f.flags) reasons.add(flag); - if (flagged.length === 0) reasons.add('no_transactions_extracted'); - if (minConfidence < CONFIDENCE_THRESHOLD) reasons.add('low_confidence'); - - const reason = Array.from(reasons).join(',') || 'needs_review'; + const reason = reviewReason(flagged, minConfidence); const payload = JSON.stringify({ minConfidence, extractor: meta.extractor, @@ -531,6 +571,14 @@ async function routeToReview( ); } +function reviewReason(flagged: FlaggedTx[], minConfidence: number): string { + const reasons = new Set(); + for (const f of flagged) for (const flag of f.flags) reasons.add(flag); + if (flagged.length === 0) reasons.add('no_transactions_extracted'); + if (minConfidence < CONFIDENCE_THRESHOLD) reasons.add('low_confidence'); + return Array.from(reasons).join(',') || 'needs_review'; +} + // --------------------------------------------------------------------------- // Small helpers // --------------------------------------------------------------------------- diff --git a/app/src/extraction/senateHtml.ts b/app/src/extraction/senateHtml.ts index 4f1069caf..d7c310ffa 100644 --- a/app/src/extraction/senateHtml.ts +++ b/app/src/extraction/senateHtml.ts @@ -231,13 +231,15 @@ function rowToParsedTx(cells: string[], map: ColumnMap, confidence: number): Par const ticker = normalizeTicker(tickerRaw); const combinedText = cells.join(' '); const capGainsCell = get(map.capGains); + const assetType = get(map.assetType).trim() || null; return { txDate: normalizeDate(get(map.date)), owner: normalizeOwner(get(map.owner), assetName), assetName: assetName || tickerRaw || '(unknown)', ticker, - assetType: get(map.assetType).trim() || null, + assetType, + assetTypeName: assetType, txType, amountMin: min, amountMax: max, diff --git a/app/src/extraction/textPdf.ts b/app/src/extraction/textPdf.ts index ae401e425..2a8a60f79 100644 --- a/app/src/extraction/textPdf.ts +++ b/app/src/extraction/textPdf.ts @@ -20,6 +20,7 @@ import { extractText, getDocumentProxy } from 'unpdf'; import type { Extractor, ExtractorInput, ExtractorResult } from '../extractors/types'; import type { Filing, Owner, ParsedTx, TxType } from '../shared/types'; +import { HOUSE_ASSET_TYPE_NAMES, houseAssetTypeCodePattern } from '../shared/assetTypes'; import { parseAmountRange } from './amounts'; import { detectOption } from './senateHtml'; @@ -80,62 +81,8 @@ const OWNER_CODES: Record = { SELF: 'self', }; -const HOUSE_ASSET_TYPE_NAMES: Record = { - '4K': '401K and Other Non-Federal Retirement Accounts', - '5C': '529 College Savings Plan', - '5F': '529 Portfolio', - '5P': '529 Prepaid Tuition Plan', - AB: 'Asset-Backed Securities', - BA: 'Bank Accounts, Money Market Accounts and CDs', - BK: 'Brokerage Accounts', - CO: 'Collectibles', - CS: 'Corporate Securities (Bonds and Notes)', - CT: 'Cryptocurrency', - DB: 'Defined Benefit Pension', - DO: 'Debts Owed to the Filer', - DS: 'Delaware Statutory Trust', - EF: 'Exchange Traded Funds (ETF)', - EQ: 'Excepted/Qualified Blind Trust', - ET: 'Exchange Traded Notes', - FA: 'Farms', - FE: 'Foreign Exchange Position (Currency)', - FN: 'Fixed Annuity', - FU: 'Futures', - GS: 'Government Securities and Agency Debt', - HE: 'Hedge Funds & Private Equity Funds (EIF)', - HN: 'Hedge Funds & Private Equity Funds (non-EIF)', - IC: 'Investment Club', - IH: 'IRA (Held in Cash)', - IP: 'Intellectual Property & Royalties', - IR: 'IRA', - MA: 'Managed Accounts (e.g., SMA and UMA)', - MF: 'Mutual Funds', - MO: 'Mineral/Oil/Solar Energy Rights', - OI: 'Ownership Interest (Holding Investments)', - OL: 'Ownership Interest (Engaged in a Trade or Business)', - OP: 'Options', - OT: 'Other', - PE: 'Pensions', - PM: 'Precious Metals', - PS: 'Stock (Not Publicly Traded)', - RE: 'Real Estate Invest. Trust (REIT)', - RF: 'REIT (EIF)', - RN: 'REIT (non-EIF)', - RP: 'Real Property', - RS: 'Restricted Stock Units (RSUs)', - SA: 'Stock Appreciation Right', - ST: 'Stocks (including ADRs)', - TR: 'Trust', - VA: 'Variable Annuity', - VI: 'Variable Insurance', - WU: 'Whole/Universal Insurance', -}; - // Asset-type bracket codes used by the House template, e.g. [ST] [OP] [GS] [4K]. -const HOUSE_ASSET_TYPE_CODE_PATTERN = Object.keys(HOUSE_ASSET_TYPE_NAMES) - .sort((a, b) => b.length - a.length) - .map(escapeRegExp) - .join('|'); +const HOUSE_ASSET_TYPE_CODE_PATTERN = houseAssetTypeCodePattern(); const ASSET_TYPE_RE = new RegExp(`\\[(${HOUSE_ASSET_TYPE_CODE_PATTERN})\\]`, 'i'); // A date in MM/DD/YYYY. const DATE_RE = /\b(\d{1,2}\/\d{1,2}\/\d{2,4})\b/g; @@ -275,10 +222,6 @@ function stripLeadingOwnerCode(value: string): string { return value.replace(/^(SP|DC|JT|SELF)\b\s*/i, '').trim(); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - function normalizeTicker(value: string | null): string | null { if (!value) return null; const ticker = value.toUpperCase().replace('/', '.'); diff --git a/app/src/shared/__tests__/assetTypes.test.ts b/app/src/shared/__tests__/assetTypes.test.ts new file mode 100644 index 000000000..2626521c8 --- /dev/null +++ b/app/src/shared/__tests__/assetTypes.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from 'vitest'; +import { + assetTypeCategoryLabel, + canonicalAssetTypeCategorySql, + canonicalizeAssetType, + houseAssetTypeCodePattern, +} from '../assetTypes'; + +describe('canonicalizeAssetType', () => { + it('maps House codes and Senate labels into the same public equity category', () => { + expect(canonicalizeAssetType('ST').category).toBe('public_equity'); + expect(canonicalizeAssetType('Stock').category).toBe('public_equity'); + expect(canonicalizeAssetType('ST').label).toBe('Stocks (including ADRs)'); + }); + + it('maps government and municipal debt across chambers', () => { + expect(canonicalizeAssetType('GS').category).toBe('fixed_income_government'); + expect(canonicalizeAssetType('Municipal Security').category).toBe('fixed_income_government'); + expect(canonicalizeAssetType('GS').categoryLabel).toBe('Government / Municipal Debt'); + }); + + it('recognizes digit-prefixed retirement and 529 House codes', () => { + for (const code of ['4K', '5C', '5F', '5P']) { + expect(canonicalizeAssetType(code).category).toBe('retirement_or_529'); + } + }); + + it('keeps private/non-public equity distinct from public stock', () => { + expect(canonicalizeAssetType('PS').category).toBe('private_equity'); + expect(canonicalizeAssetType('Non-Public Stock').category).toBe('private_equity'); + }); + + it('falls back from explicit option flag when the raw type is missing', () => { + expect(canonicalizeAssetType(null, null, { isOption: true }).category).toBe('option'); + }); + + it('lets disclosed type labels override the option flag', () => { + expect(canonicalizeAssetType('Stock', null, { isOption: true }).category).toBe('public_equity'); + }); + + it('keeps present but unmapped values separate from missing values', () => { + expect(canonicalizeAssetType('Bespoke Security').category).toBe('other'); + }); + + it('treats blank and PDF placeholder rows as unknown', () => { + expect(canonicalizeAssetType(null).category).toBe('unknown'); + expect(canonicalizeAssetType('PDF Disclosed Filing').category).toBe('unknown'); + expect(canonicalizeAssetType('N/A').category).toBe('unknown'); + expect(canonicalizeAssetType('-').category).toBe('unknown'); + expect(canonicalizeAssetType('--').category).toBe('unknown'); + expect(assetTypeCategoryLabel('unknown')).toBe('Unknown'); + }); + + it('exposes a House-code pattern that includes digit-prefixed codes', () => { + const pattern = houseAssetTypeCodePattern(); + expect(new RegExp(`^(?:${pattern})$`).test('4K')).toBe(true); + expect(new RegExp(`^(?:${pattern})$`).test('GS')).toBe(true); + }); +}); + +describe('canonicalAssetTypeCategorySql', () => { + it('generates a SQL CASE expression for grouped analytics', () => { + const sql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); + expect(sql).toContain("THEN 'public_equity'"); + expect(sql).toContain("THEN 'fixed_income_government'"); + expect(sql).toContain('t.is_option = 1'); + }); + + it('matches the canonical fallback order in SQL', () => { + const sql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); + expect(sql.indexOf("WHEN t.is_option = 1 THEN 'option'")).toBeGreaterThan( + sql.indexOf("THEN 'public_equity'"), + ); + expect(sql).toContain( + "WHEN lower(trim(coalesce(t.asset_type, ''))) = '' AND lower(trim(coalesce(t.asset_type_name, ''))) = '' THEN 'unknown'", + ); + expect(sql).not.toContain("lower(trim(coalesce(t.asset_type_name, ''))) = '' OR"); + }); +}); diff --git a/app/src/shared/assetTypes.ts b/app/src/shared/assetTypes.ts new file mode 100644 index 000000000..22ddcdc1f --- /dev/null +++ b/app/src/shared/assetTypes.ts @@ -0,0 +1,383 @@ +/** + * Shared instrument-type canonicalization. + * + * Keep raw disclosure values in transactions.asset_type / asset_type_name, then + * compute this layer at read/export time so House bracket codes and Senate eFD + * labels can roll up together without erasing provenance. + */ + +export type AssetTypeCategory = + | 'public_equity' + | 'private_equity' + | 'option' + | 'fund' + | 'fixed_income_government' + | 'fixed_income_corporate' + | 'fixed_income_asset_backed' + | 'cash' + | 'retirement_or_529' + | 'real_estate' + | 'private_fund' + | 'business_interest' + | 'crypto' + | 'insurance_annuity' + | 'trust' + | 'commodity_collectible' + | 'derivative' + | 'intellectual_property' + | 'receivable' + | 'other_security' + | 'other' + | 'unknown'; + +export type AssetTypeSource = 'house_code' | 'label' | 'option_flag' | 'missing' | 'unknown'; + +export interface CanonicalAssetType { + rawType: string | null; + rawName: string | null; + code: string | null; + label: string; + category: AssetTypeCategory; + categoryLabel: string; + source: AssetTypeSource; +} + +export const HOUSE_ASSET_TYPE_NAMES: Record = { + '4K': '401K and Other Non-Federal Retirement Accounts', + '5C': '529 College Savings Plan', + '5F': '529 Portfolio', + '5P': '529 Prepaid Tuition Plan', + AB: 'Asset-Backed Securities', + BA: 'Bank Accounts, Money Market Accounts and CDs', + BK: 'Brokerage Accounts', + CO: 'Collectibles', + CS: 'Corporate Securities (Bonds and Notes)', + CT: 'Cryptocurrency', + DB: 'Defined Benefit Pension', + DO: 'Debts Owed to the Filer', + DS: 'Delaware Statutory Trust', + EF: 'Exchange Traded Funds (ETF)', + EQ: 'Excepted/Qualified Blind Trust', + ET: 'Exchange Traded Notes', + FA: 'Farms', + FE: 'Foreign Exchange Position (Currency)', + FN: 'Fixed Annuity', + FU: 'Futures', + GS: 'Government Securities and Agency Debt', + HE: 'Hedge Funds & Private Equity Funds (EIF)', + HN: 'Hedge Funds & Private Equity Funds (non-EIF)', + IC: 'Investment Club', + IH: 'IRA (Held in Cash)', + IP: 'Intellectual Property & Royalties', + IR: 'IRA', + MA: 'Managed Accounts (e.g., SMA and UMA)', + MF: 'Mutual Funds', + MO: 'Mineral/Oil/Solar Energy Rights', + OI: 'Ownership Interest (Holding Investments)', + OL: 'Ownership Interest (Engaged in a Trade or Business)', + OP: 'Options', + OT: 'Other', + PE: 'Pensions', + PM: 'Precious Metals', + PS: 'Stock (Not Publicly Traded)', + RE: 'Real Estate Invest. Trust (REIT)', + RF: 'REIT (EIF)', + RN: 'REIT (non-EIF)', + RP: 'Real Property', + RS: 'Restricted Stock Units (RSUs)', + SA: 'Stock Appreciation Right', + ST: 'Stocks (including ADRs)', + TR: 'Trust', + VA: 'Variable Annuity', + VI: 'Variable Insurance', + WU: 'Whole/Universal Insurance', +}; + +export const HOUSE_ASSET_TYPE_CATEGORIES: Record = { + '4K': 'retirement_or_529', + '5C': 'retirement_or_529', + '5F': 'retirement_or_529', + '5P': 'retirement_or_529', + AB: 'fixed_income_asset_backed', + BA: 'cash', + BK: 'cash', + CO: 'commodity_collectible', + CS: 'fixed_income_corporate', + CT: 'crypto', + DB: 'retirement_or_529', + DO: 'receivable', + DS: 'trust', + EF: 'fund', + EQ: 'trust', + ET: 'fund', + FA: 'commodity_collectible', + FE: 'derivative', + FN: 'insurance_annuity', + FU: 'derivative', + GS: 'fixed_income_government', + HE: 'private_fund', + HN: 'private_fund', + IC: 'business_interest', + IH: 'retirement_or_529', + IP: 'intellectual_property', + IR: 'retirement_or_529', + MA: 'fund', + MF: 'fund', + MO: 'commodity_collectible', + OI: 'business_interest', + OL: 'business_interest', + OP: 'option', + OT: 'other', + PE: 'retirement_or_529', + PM: 'commodity_collectible', + PS: 'private_equity', + RE: 'real_estate', + RF: 'real_estate', + RN: 'real_estate', + RP: 'real_estate', + RS: 'derivative', + SA: 'derivative', + ST: 'public_equity', + TR: 'trust', + VA: 'insurance_annuity', + VI: 'insurance_annuity', + WU: 'insurance_annuity', +}; + +export const ASSET_TYPE_CATEGORY_LABELS: Record = { + public_equity: 'Public Equity', + private_equity: 'Private Equity', + option: 'Options', + fund: 'Funds / ETFs / REITs', + fixed_income_government: 'Government / Municipal Debt', + fixed_income_corporate: 'Corporate Debt', + fixed_income_asset_backed: 'Asset-Backed Securities', + cash: 'Cash / Bank Accounts', + retirement_or_529: 'Retirement / 529 Accounts', + real_estate: 'Real Estate', + private_fund: 'Private Funds', + business_interest: 'Business Interests', + crypto: 'Crypto', + insurance_annuity: 'Insurance / Annuities', + trust: 'Trusts', + commodity_collectible: 'Commodities / Collectibles', + derivative: 'Derivatives / Rights', + intellectual_property: 'Intellectual Property', + receivable: 'Receivables', + other_security: 'Other Securities', + other: 'Other', + unknown: 'Unknown', +}; + +const LABEL_CATEGORY_ALIASES: Record = { + '401k and other non federal retirement accounts': 'retirement_or_529', + '401k and other non-federal retirement accounts': 'retirement_or_529', + '529 college savings plan': 'retirement_or_529', + '529 portfolio': 'retirement_or_529', + '529 prepaid tuition plan': 'retirement_or_529', + 'asset backed securities': 'fixed_income_asset_backed', + 'asset-backed securities': 'fixed_income_asset_backed', + 'bank accounts money market accounts and cds': 'cash', + 'brokerage accounts': 'cash', + cash: 'cash', + cd: 'cash', + cds: 'cash', + collectibles: 'commodity_collectible', + 'corporate bond': 'fixed_income_corporate', + 'corporate bonds': 'fixed_income_corporate', + 'corporate debt': 'fixed_income_corporate', + 'corporate securities bonds and notes': 'fixed_income_corporate', + cryptocurrency: 'crypto', + crypto: 'crypto', + 'defined benefit pension': 'retirement_or_529', + 'debts owed to the filer': 'receivable', + 'delaware statutory trust': 'trust', + 'exchange traded funds etf': 'fund', + etf: 'fund', + 'exchange traded notes': 'fund', + etn: 'fund', + 'excepted qualified blind trust': 'trust', + farms: 'commodity_collectible', + 'foreign exchange position currency': 'derivative', + 'fixed annuity': 'insurance_annuity', + futures: 'derivative', + 'government securities and agency debt': 'fixed_income_government', + 'municipal security': 'fixed_income_government', + muni: 'fixed_income_government', + 'government municipal debt': 'fixed_income_government', + 'hedge funds private equity funds eif': 'private_fund', + 'hedge funds private equity funds non eif': 'private_fund', + 'private fund': 'private_fund', + 'private funds': 'private_fund', + 'investment club': 'business_interest', + 'ira held in cash': 'retirement_or_529', + ira: 'retirement_or_529', + 'intellectual property royalties': 'intellectual_property', + 'managed accounts e g sma and uma': 'fund', + 'managed accounts sma and uma': 'fund', + 'mutual funds': 'fund', + 'mutual fund': 'fund', + 'mineral oil solar energy rights': 'commodity_collectible', + 'ownership interest holding investments': 'business_interest', + 'ownership interest engaged in a trade or business': 'business_interest', + option: 'option', + options: 'option', + 'stock option': 'option', + 'other securities': 'other_security', + other: 'other', + pensions: 'retirement_or_529', + pension: 'retirement_or_529', + 'precious metals': 'commodity_collectible', + 'stock not publicly traded': 'private_equity', + 'non public stock': 'private_equity', + 'non-public stock': 'private_equity', + 'real estate invest trust reit': 'real_estate', + reit: 'real_estate', + 'reit eif': 'real_estate', + 'reit non eif': 'real_estate', + 'real property': 'real_estate', + 'restricted stock units rsus': 'derivative', + rsu: 'derivative', + rsus: 'derivative', + 'stock appreciation right': 'derivative', + stock: 'public_equity', + stocks: 'public_equity', + 'stocks including adrs': 'public_equity', + equity: 'public_equity', + trust: 'trust', + 'variable annuity': 'insurance_annuity', + 'variable insurance': 'insurance_annuity', + 'whole universal insurance': 'insurance_annuity', +}; + +const UNKNOWN_LABELS = new Set(['', 'unknown', 'pdf disclosed filing', 'n/a', 'n a', 'na', '--', '-']); + +export function houseAssetTypeCodePattern(): string { + return Object.keys(HOUSE_ASSET_TYPE_NAMES).sort((a, b) => b.length - a.length).map(escapeRegExp).join('|'); +} + +export function assetTypeCategoryLabel(category: AssetTypeCategory): string { + return ASSET_TYPE_CATEGORY_LABELS[category] ?? ASSET_TYPE_CATEGORY_LABELS.unknown; +} + +export function isAssetTypeCategory(value: string | null | undefined): value is AssetTypeCategory { + return !!value && Object.prototype.hasOwnProperty.call(ASSET_TYPE_CATEGORY_LABELS, value); +} + +export function canonicalizeAssetType( + rawType: string | null | undefined, + rawName?: string | null, + opts?: { isOption?: boolean | null; assetName?: string | null }, +): CanonicalAssetType { + const type = cleanNullable(rawType); + const name = cleanNullable(rawName); + const upper = type?.toUpperCase() ?? null; + + if (upper && HOUSE_ASSET_TYPE_NAMES[upper]) { + const category = HOUSE_ASSET_TYPE_CATEGORIES[upper] ?? 'other'; + return { + rawType: type, + rawName: name, + code: upper, + label: HOUSE_ASSET_TYPE_NAMES[upper], + category, + categoryLabel: assetTypeCategoryLabel(category), + source: 'house_code', + }; + } + + for (const value of [name, type, opts?.assetName ?? null]) { + if (isUnknownAssetTypeValue(value)) { + return canonical('unknown', type, name, value, 'missing'); + } + const key = normalizeAssetTypeKey(value); + if (!key) continue; + const category = LABEL_CATEGORY_ALIASES[key]; + if (category) return canonical(category, type, name, value, 'label'); + } + + if (opts?.isOption) return canonical('option', type, name, type ?? name ?? 'Option', 'option_flag'); + if (!type && !name) return canonical('unknown', null, null, null, 'missing'); + return canonical('other', type, name, type ?? name, 'unknown'); +} + +export function canonicalAssetTypeCategorySql( + rawTypeExpr: string, + rawNameExpr = 'NULL', + isOptionExpr?: string, +): string { + const raw = `lower(trim(coalesce(${rawTypeExpr}, '')))`; + const name = `lower(trim(coalesce(${rawNameExpr}, '')))`; + const upper = `upper(trim(coalesce(${rawTypeExpr}, '')))`; + const optionFlag = isOptionExpr ? `WHEN ${isOptionExpr} = 1 THEN 'option'` : ''; + const houseWhen = Object.entries(HOUSE_ASSET_TYPE_CATEGORIES) + .map(([code, category]) => `WHEN ${upper} = ${sqlQuote(code)} THEN ${sqlQuote(category)}`) + .join(' '); + const houseLabelWhen = Object.entries(HOUSE_ASSET_TYPE_CATEGORIES) + .map(([code, category]) => { + const label = HOUSE_ASSET_TYPE_NAMES[code].toLowerCase(); + return `WHEN ${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)} THEN ${sqlQuote(category)}`; + }) + .join(' '); + const labelWhen = Object.entries(LABEL_CATEGORY_ALIASES) + .map( + ([label, category]) => + `WHEN ${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)} THEN ${sqlQuote(category)}`, + ) + .join(' '); + const unknownWhen = Array.from(UNKNOWN_LABELS) + .filter(Boolean) + .map((label) => `${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)}`) + .join(' OR '); + const unknownCase = unknownWhen ? `WHEN ${unknownWhen} THEN 'unknown'` : ''; + return `(CASE ${houseWhen} ${houseLabelWhen} ${labelWhen} ${unknownCase} ${optionFlag} WHEN ${raw} = '' AND ${name} = '' THEN 'unknown' ELSE 'other' END)`; +} + +function canonical( + category: AssetTypeCategory, + rawType: string | null, + rawName: string | null, + label: string | null | undefined, + source: AssetTypeSource, +): CanonicalAssetType { + const categoryLabel = assetTypeCategoryLabel(category); + return { + rawType, + rawName, + code: null, + label: cleanNullable(label) ?? categoryLabel, + category, + categoryLabel, + source, + }; +} + +function normalizeAssetTypeKey(value: string | null | undefined): string { + return (value ?? '') + .toLowerCase() + .replace(/&/g, ' ') + .replace(/[^a-z0-9]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function isUnknownAssetTypeValue(value: string | null | undefined): boolean { + if (value == null) return false; + const raw = value.trim().toLowerCase(); + if (UNKNOWN_LABELS.has(raw)) return true; + const key = normalizeAssetTypeKey(value); + return key ? UNKNOWN_LABELS.has(key) : false; +} + +function cleanNullable(value: string | null | undefined): string | null { + const cleaned = (value ?? '').trim(); + return cleaned ? cleaned : null; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function sqlQuote(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/app/src/shared/ingestionDecisions.ts b/app/src/shared/ingestionDecisions.ts new file mode 100644 index 000000000..28b7c089c --- /dev/null +++ b/app/src/shared/ingestionDecisions.ts @@ -0,0 +1,138 @@ +import { all, parseJson, run, type SqlParam } from './db'; +import { uuid } from './ids'; + +export type IngestionDecisionAction = + | 'auto_published' + | 'review_opened' + | 'confirmed' + | 'manual' + | 'rejected' + | 'unpublished' + | 'agreement_published'; + +export type IngestionDecisionSource = 'pipeline' | 'admin' | 'agreement'; + +export interface IngestionDecisionInput { + docId: string; + action: IngestionDecisionAction; + source: IngestionDecisionSource; + actor?: string | null; + reason?: string | null; + payload?: Record | null; + transactionIds?: string[]; + createdAt?: string; +} + +interface IngestionDecisionRow { + id: string; + doc_id: string; + action: string; + source: string; + actor: string | null; + reason: string | null; + payload: string | null; + transaction_ids: string | null; + created_at: string; + chamber?: string | null; + ingest_status?: string | null; + source_url?: string | null; +} + +export interface ListedIngestionDecision { + id: string; + docId: string; + action: string; + source: string; + actor: string | null; + reason: string | null; + payload: Record | null; + transactionIds: string[]; + createdAt: string; + chamber: string | null; + ingestStatus: string | null; + sourceUrl: string | null; +} + +/** + * Best-effort audit write. The decision trail should never block ingestion or + * admin remediation if a deployment reaches code before the D1 migration. + */ +export async function recordIngestionDecision( + db: D1Database, + input: IngestionDecisionInput, +): Promise { + const id = uuid(); + const createdAt = input.createdAt ?? new Date().toISOString(); + try { + await run( + db, + `INSERT INTO ingestion_decisions + (id, doc_id, action, source, actor, reason, payload, transaction_ids, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + id, + input.docId, + input.action, + input.source, + input.actor ?? null, + input.reason ?? null, + input.payload ? JSON.stringify(input.payload) : null, + JSON.stringify(input.transactionIds ?? []), + createdAt, + ], + ); + return id; + } catch (err) { + console.warn( + 'ingestion decision audit write skipped:', + input.docId, + input.action, + (err as Error).message, + ); + return null; + } +} + +export async function listIngestionDecisions( + db: D1Database, + opts: { limit?: number; docId?: string | null } = {}, +): Promise { + const requestedLimit = Number.isFinite(opts.limit) ? Math.floor(opts.limit as number) : 100; + const limit = Math.min(Math.max(requestedLimit, 1), 200); + const where: string[] = []; + const params: SqlParam[] = []; + if (opts.docId) { + where.push('d.doc_id = ?'); + params.push(opts.docId); + } + params.push(limit); + + const rows = await all( + db, + `SELECT + d.id, d.doc_id, d.action, d.source, d.actor, d.reason, d.payload, + d.transaction_ids, d.created_at, + f.chamber, f.ingest_status, f.source_url + FROM ingestion_decisions d + LEFT JOIN filings f ON f.doc_id = d.doc_id + ${where.length ? `WHERE ${where.join(' AND ')}` : ''} + ORDER BY d.created_at DESC + LIMIT ?`, + params, + ); + + return rows.map((row) => ({ + id: row.id, + docId: row.doc_id, + action: row.action, + source: row.source, + actor: row.actor ?? null, + reason: row.reason ?? null, + payload: parseJson | null>(row.payload, null), + transactionIds: parseJson(row.transaction_ids, []), + createdAt: row.created_at, + chamber: row.chamber ?? null, + ingestStatus: row.ingest_status ?? null, + sourceUrl: row.source_url ?? null, + })); +} diff --git a/app/src/shared/types.ts b/app/src/shared/types.ts index 178f673dc..4c241b8af 100644 --- a/app/src/shared/types.ts +++ b/app/src/shared/types.ts @@ -6,6 +6,8 @@ * as the source of truth — downstream agents implement against these shapes. */ +import type { AssetTypeCategory } from './assetTypes'; + // --------------------------------------------------------------------------- // Primitive unions / enums // --------------------------------------------------------------------------- @@ -96,6 +98,8 @@ export interface Transaction { ticker: string | null; assetType: string | null; assetTypeName?: string | null; + assetTypeCategory?: AssetTypeCategory; + assetTypeCategoryLabel?: string; txType: TxType; amountMin: number | null; amountMax: number | null; @@ -253,7 +257,14 @@ export interface ClientTrade { companyName: string | null; /** Same-origin cached logo proxy URL for the ticker, or null when no ticker is resolved. */ logoUrl: string | null; + /** Raw disclosure asset type/code as stored in transactions.asset_type. */ type: string | null; + /** Expanded raw type name when available, e.g. House code label. */ + typeName: string | null; + /** Cross-chamber canonical instrument category, computed from raw type/code. */ + typeCategory: AssetTypeCategory; + /** Human label for typeCategory. */ + typeCategoryLabel: string; sector: string | null; marketCapBucket: string | null; }; diff --git a/app/src/ui/__tests__/dashboardHtml.test.ts b/app/src/ui/__tests__/dashboardHtml.test.ts index f49b490e5..bce0d9661 100644 --- a/app/src/ui/__tests__/dashboardHtml.test.ts +++ b/app/src/ui/__tests__/dashboardHtml.test.ts @@ -203,6 +203,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('function useModelRows('); expect(DASHBOARD_HTML).toContain('function openReviewEditor('); expect(DASHBOARD_HTML).toContain('Review / Confirm'); + expect(DASHBOARD_HTML).toContain('Decision History'); + expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=100'"); + expect(DASHBOARD_HTML).toContain('function renderDecisionHistory('); + expect(DASHBOARD_HTML).toContain('var DECISIONS'); expect(DASHBOARD_HTML).toContain('Use This Model'); expect(DASHBOARD_HTML).toContain('Bake-Off Runs ('); expect(DASHBOARD_HTML).toContain('Queued Extracted Rows'); @@ -210,8 +214,16 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('REVIEW_AMOUNT_BRACKETS'); expect(DASHBOARD_HTML).toContain('class="me-bracket"'); expect(DASHBOARD_HTML).toContain('class="me-asset-type"'); - expect(DASHBOARD_HTML).toContain('Stocks (ST)'); - expect(DASHBOARD_HTML).toContain('Other (OT)'); + expect(DASHBOARD_HTML).toContain('HOUSE_REVIEW_ASSET_TYPES'); + expect(DASHBOARD_HTML).toContain('SENATE_REVIEW_ASSET_TYPES'); + expect(DASHBOARD_HTML).toContain('function reviewAssetTypeDatalistId('); + expect(DASHBOARD_HTML).toContain('function reviewNormalizeAssetTypeValue('); + expect(DASHBOARD_HTML).toContain('list="'); + expect(DASHBOARD_HTML).toContain('5P'); + expect(DASHBOARD_HTML).toContain('Municipal Security'); + expect(DASHBOARD_HTML).toContain('Stock Option'); + expect(DASHBOARD_HTML).toContain('assetTypeName: reviewAssetTypeName(assetType) || null'); + expect(DASHBOARD_HTML).toContain("tr.setAttribute('data-chamber', chamber || '')"); expect(DASHBOARD_HTML).toContain('class="me-option"'); expect(DASHBOARD_HTML).toContain('Option Contract'); expect(DASHBOARD_HTML).toContain('class="me-cap"'); diff --git a/app/src/ui/dashboardHtml.ts b/app/src/ui/dashboardHtml.ts index d5633e201..4a0484cff 100644 --- a/app/src/ui/dashboardHtml.ts +++ b/app/src/ui/dashboardHtml.ts @@ -438,6 +438,8 @@ export const DASHBOARD_HTML = /* html */ ` .me-row { margin: 8px 0; display:grid; grid-template-columns:minmax(80px,.6fr) 130px 165px 150px 125px 155px minmax(220px,1.8fr) minmax(210px,.9fr); gap:8px; align-items:center; } .me-row input, .me-row select { min-height:34px; width:100%; min-width:0; } .me-row .me-asset { width:100%; } + .me-asset-type-wrap { min-width:0; } + .me-asset-type-category { display:block; margin-top:2px; color:var(--text-dim); font-size:11px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .me-flags { display:flex; align-items:center; gap:6px 10px; flex-wrap:wrap; min-width:0; } .me-check { display:inline-flex; align-items:center; gap:4px; color:var(--text-dim); font-size:12px; white-space:nowrap; } .me-check input { min-height:0; width:auto; } @@ -1284,6 +1286,13 @@ export const DASHBOARD_HTML = /* html */ `

Confirm promotes the read to the live feed; Manual lets you hand-key the rows (recorded as source=manual) when the automated read is wrong or too low-confidence; Reject discards it. Models / readings come from extraction_runs (populated by POST /api/admin/bakeoff). POST /api/admin/review/:docId {decision}

+
+

Decision History

+ + + +
TimeDocActionSourceReasonRows
+
@@ -1470,6 +1479,7 @@ export const DASHBOARD_HTML = /* html */ ` /* ============================ STATE ============================ */ var TRADES = []; // live transactions (newest first) var REVIEW = []; // review-queue items +var DECISIONS = []; // ingestion decision audit rows var REVIEW_RUNS = {}; // docId -> full extraction runs loaded on demand var SCHEDULE = []; // PollWindow[] var aggressive = false; @@ -1543,25 +1553,132 @@ function assetClassLabel(c) { if (!s) return ''; return ASSET_CLASS_UP[s.toLowerCase()] || (s.charAt(0).toUpperCase() + s.slice(1)); } -/* Friendlier label for STOCK Act asset-type codes (ST = Stocks, etc.). */ -var ASSET_TYPE_LABEL = { ST: 'Stocks', OP: 'Options', GS: 'Govt Securities', CS: 'Corporate Bonds', EF: 'Funds / ETFs', MF: 'Mutual Funds', OT: 'Other', PE: 'Private Equity', RP: 'Real Property', Unknown: 'Unclassified' }; -var REVIEW_ASSET_TYPES = [ - ['ST', 'Stocks (ST)'], - ['OP', 'Options (OP)'], - ['GS', 'Govt Securities (GS)'], - ['CS', 'Corporate Bonds (CS)'], - ['EF', 'Funds / ETFs (EF)'], - ['MF', 'Mutual Funds (MF)'], - ['OT', 'Other (OT)'], - ['PE', 'Private Equity (PE)'], - ['RP', 'Real Property (RP)'], - ['Unknown', 'Unclassified'] +/* Friendlier labels for raw STOCK Act asset-type codes / Senate eFD labels. */ +var HOUSE_REVIEW_ASSET_TYPES = [ + ['4K', '401K and Other Non-Federal Retirement Accounts', 'Retirement / 529 Accounts'], + ['5C', '529 College Savings Plan', 'Retirement / 529 Accounts'], + ['5F', '529 Portfolio', 'Retirement / 529 Accounts'], + ['5P', '529 Prepaid Tuition Plan', 'Retirement / 529 Accounts'], + ['AB', 'Asset-Backed Securities', 'Asset-Backed Securities'], + ['BA', 'Bank Accounts, Money Market Accounts and CDs', 'Cash / Bank Accounts'], + ['BK', 'Brokerage Accounts', 'Cash / Bank Accounts'], + ['CO', 'Collectibles', 'Commodities / Collectibles'], + ['CS', 'Corporate Securities (Bonds and Notes)', 'Corporate Debt'], + ['CT', 'Cryptocurrency', 'Crypto'], + ['DB', 'Defined Benefit Pension', 'Retirement / 529 Accounts'], + ['DO', 'Debts Owed to the Filer', 'Receivables'], + ['DS', 'Delaware Statutory Trust', 'Trusts'], + ['EF', 'Exchange Traded Funds (ETF)', 'Funds / ETFs / REITs'], + ['EQ', 'Excepted/Qualified Blind Trust', 'Trusts'], + ['ET', 'Exchange Traded Notes', 'Funds / ETFs / REITs'], + ['FA', 'Farms', 'Commodities / Collectibles'], + ['FE', 'Foreign Exchange Position (Currency)', 'Derivatives / Rights'], + ['FN', 'Fixed Annuity', 'Insurance / Annuities'], + ['FU', 'Futures', 'Derivatives / Rights'], + ['GS', 'Government Securities and Agency Debt', 'Government / Municipal Debt'], + ['HE', 'Hedge Funds & Private Equity Funds (EIF)', 'Private Funds'], + ['HN', 'Hedge Funds & Private Equity Funds (non-EIF)', 'Private Funds'], + ['IC', 'Investment Club', 'Business Interests'], + ['IH', 'IRA (Held in Cash)', 'Retirement / 529 Accounts'], + ['IP', 'Intellectual Property & Royalties', 'Intellectual Property'], + ['IR', 'IRA', 'Retirement / 529 Accounts'], + ['MA', 'Managed Accounts (e.g., SMA and UMA)', 'Funds / ETFs / REITs'], + ['MF', 'Mutual Funds', 'Funds / ETFs / REITs'], + ['MO', 'Mineral/Oil/Solar Energy Rights', 'Commodities / Collectibles'], + ['OI', 'Ownership Interest (Holding Investments)', 'Business Interests'], + ['OL', 'Ownership Interest (Engaged in a Trade or Business)', 'Business Interests'], + ['OP', 'Options', 'Options'], + ['OT', 'Other', 'Other'], + ['PE', 'Pensions', 'Retirement / 529 Accounts'], + ['PM', 'Precious Metals', 'Commodities / Collectibles'], + ['PS', 'Stock (Not Publicly Traded)', 'Private Equity'], + ['RE', 'Real Estate Invest. Trust (REIT)', 'Real Estate'], + ['RF', 'REIT (EIF)', 'Real Estate'], + ['RN', 'REIT (non-EIF)', 'Real Estate'], + ['RP', 'Real Property', 'Real Estate'], + ['RS', 'Restricted Stock Units (RSUs)', 'Derivatives / Rights'], + ['SA', 'Stock Appreciation Right', 'Derivatives / Rights'], + ['ST', 'Stocks (including ADRs)', 'Public Equity'], + ['TR', 'Trust', 'Trusts'], + ['VA', 'Variable Annuity', 'Insurance / Annuities'], + ['VI', 'Variable Insurance', 'Insurance / Annuities'], + ['WU', 'Whole/Universal Insurance', 'Insurance / Annuities'] +]; +var SENATE_REVIEW_ASSET_TYPES = [ + ['Stock', 'Stock', 'Public Equity'], + ['Stock Option', 'Stock Option', 'Options'], + ['Municipal Security', 'Municipal Security', 'Government / Municipal Debt'], + ['Corporate Bond', 'Corporate Bond', 'Corporate Debt'], + ['Other Securities', 'Other Securities', 'Other Securities'], + ['Non-Public Stock', 'Non-Public Stock', 'Private Equity'] ]; +var REVIEW_UNKNOWN_ASSET_TYPES = [['Unknown', 'Unclassified', 'Unknown']]; +var REVIEW_ASSET_TYPES = HOUSE_REVIEW_ASSET_TYPES.concat(SENATE_REVIEW_ASSET_TYPES).concat(REVIEW_UNKNOWN_ASSET_TYPES); +var ASSET_TYPE_LABEL = {}; +var ASSET_TYPE_CATEGORY_LABEL = {}; +REVIEW_ASSET_TYPES.forEach(function (pair) { + ASSET_TYPE_LABEL[pair[0]] = pair[1]; + ASSET_TYPE_LABEL[String(pair[0]).toUpperCase()] = pair[1]; + ASSET_TYPE_CATEGORY_LABEL[pair[0]] = pair[2]; + ASSET_TYPE_CATEGORY_LABEL[String(pair[0]).toUpperCase()] = pair[2]; +}); function assetTypeLabel(t) { var s = String(t == null ? '' : t).trim(); if (!s) return 'Unclassified'; return ASSET_TYPE_LABEL[s] || ASSET_TYPE_LABEL[s.toUpperCase()] || s; } +function reviewAssetTypeName(t) { + var s = String(t == null ? '' : t).trim(); + if (!s || s.toLowerCase() === 'unknown') return ''; + return ASSET_TYPE_LABEL[s] || ASSET_TYPE_LABEL[s.toUpperCase()] || ''; +} +function reviewNormalizeAssetTypeValue(t) { + var s = String(t == null ? '' : t).trim(); + if (!s) return ''; + var upper = s.toUpperCase(); + for (var i = 0; i < HOUSE_REVIEW_ASSET_TYPES.length; i++) { + if (HOUSE_REVIEW_ASSET_TYPES[i][0] === upper) return upper; + } + var lower = s.toLowerCase(); + for (var j = 0; j < SENATE_REVIEW_ASSET_TYPES.length; j++) { + if (String(SENATE_REVIEW_ASSET_TYPES[j][0]).toLowerCase() === lower) return SENATE_REVIEW_ASSET_TYPES[j][0]; + } + if (lower === 'unknown' || lower === 'unclassified') return 'Unknown'; + return s; +} +function reviewAssetTypeCategoryLabel(t) { + var s = String(t == null ? '' : t).trim(); + if (!s) return ''; + return ASSET_TYPE_CATEGORY_LABEL[s] || ASSET_TYPE_CATEGORY_LABEL[s.toUpperCase()] || ''; +} +function reviewAssetPairsForChamber(chamber) { + var c = String(chamber == null ? '' : chamber).toLowerCase(); + if (c === 'house') return HOUSE_REVIEW_ASSET_TYPES.concat(REVIEW_UNKNOWN_ASSET_TYPES); + if (c === 'senate') return SENATE_REVIEW_ASSET_TYPES.concat(REVIEW_UNKNOWN_ASSET_TYPES); + var seen = {}; + return REVIEW_ASSET_TYPES.filter(function (pair) { + var key = String(pair[0]).toLowerCase(); + if (seen[key]) return false; + seen[key] = true; + return true; + }); +} +function reviewAssetTypeDatalistId(chamber) { + var c = String(chamber == null ? '' : chamber).toLowerCase(); + return c === 'house' ? 'review-asset-types-house' : c === 'senate' ? 'review-asset-types-senate' : 'review-asset-types-all'; +} +function ensureReviewAssetTypeDatalists() { + if (document.getElementById('review-asset-types-all')) return; + ['house', 'senate', 'all'].forEach(function (chamber) { + var dl = document.createElement('datalist'); + dl.id = reviewAssetTypeDatalistId(chamber); + dl.innerHTML = reviewAssetPairsForChamber(chamber).map(function (pair) { + var label = pair[0] === pair[1] ? pair[2] : pair[1] + ' - ' + pair[2]; + return ''; + }).join(''); + document.body.appendChild(dl); + }); +} /* Normalize a date string to YYYY-MM-DD without timezone drift. Accepts ISO ("2026-06-15...") and US ("6/15/2026") forms (Senate filings use the latter). */ function toISODate(s) { @@ -2456,11 +2573,70 @@ function loadReview() { // API HOOK: GET /api/admin/review-queue?resolved= return fetch('/api/admin/review-queue?resolved=' + REVIEW_RESOLVED, { headers: adminHeaders() }) .then(okOrThrow) - .then(function (data) { REVIEW = data.items || []; renderReview(); }) + .then(function (data) { REVIEW = data.items || []; renderReview(); loadDecisionHistory(); }) .catch(function (e) { el('reviewBody').innerHTML = stateRow(6, isAuthError(e) ? ADMIN_MOVED_MSG : ('Could not load review queue: ' + e.message)); }); } +function loadDecisionHistory() { + // API HOOK: GET /api/admin/ingestion-decisions + return fetch('/api/admin/ingestion-decisions?limit=100', { headers: adminHeaders() }) + .then(okOrThrow) + .then(function (data) { + DECISIONS = data.items || []; + renderDecisionHistory(data.available !== false); + }) + .catch(function (e) { + var body = el('decisionBody'); + if (body) body.innerHTML = stateRow(6, isAuthError(e) ? ADMIN_MOVED_MSG : ('Could not load decision history: ' + e.message)); + }); +} +function decisionActionLabel(action) { + return String(action || '').replace(/_/g, ' '); +} +function decisionRowsText(d) { + var ids = d && Array.isArray(d.transactionIds) ? d.transactionIds : []; + if (!ids.length) return '—'; + return ids.length + ' row' + (ids.length === 1 ? '' : 's'); +} +function decisionReasonText(d) { + var reason = d && d.reason ? String(d.reason) : ''; + var payload = d && d.payload && typeof d.payload === 'object' ? d.payload : null; + var bits = []; + if (reason) bits.push(reason.replace(/_/g, ' ')); + if (payload && typeof payload.minConfidence === 'number') bits.push('conf ' + Math.round(payload.minConfidence * 100) + '%'); + if (payload && typeof payload.inserted === 'number') bits.push('inserted ' + payload.inserted); + if (payload && typeof payload.deprecatedTransactions === 'number') bits.push('retracted ' + payload.deprecatedTransactions); + return bits.join(' · ') || '—'; +} +function decisionDocHtml(d) { + var docId = d.docId || ''; + var url = safeDocUrl(d.sourceUrl); + if (!url) return '' + esc(docId) + ''; + return '' + esc(docId) + ''; +} +function renderDecisionHistory(available) { + var body = el('decisionBody'); + if (!body) return; + if (!available) { + body.innerHTML = stateRow(6, 'Decision history is not migrated yet.'); + return; + } + if (!DECISIONS.length) { + body.innerHTML = stateRow(6, 'No decisions recorded yet.'); + return; + } + body.innerHTML = DECISIONS.map(function (d) { + return '' + + '' + esc(dateTimeText(d.createdAt)) + '' + + '' + decisionDocHtml(d) + '' + + '' + statusBadge(decisionActionLabel(d.action)) + '' + + '' + esc([d.source || '', d.actor || ''].filter(Boolean).join(' · ')) + '' + + '' + esc(decisionReasonText(d)) + '' + + '' + esc(decisionRowsText(d)) + '' + + ''; + }).join(''); +} /* Translate review reason codes + payload into plain English for non-engineers. */ var REASON_LABELS = { low_confidence: 'Automated read below publish threshold', @@ -2539,6 +2715,7 @@ function normalizeReviewEdit(t, sourceLabel) { txDate: String(t.txDate || '').slice(0, 10) || null, owner: owner, assetType: cleanAsset(t.assetType || ''), + assetTypeName: cleanAsset(t.assetTypeName || ''), isOption: Boolean(t.isOption), capGainsOver200: Boolean(t.capGainsOver200), confidence: t.confidence == null ? null : n(t.confidence), @@ -2713,6 +2890,7 @@ function resolveReview(docId, decision) { .then(function () { if (isUnpublish) { loadReview(); } // item returns to pending; reload current tab else { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); renderReview(); } + loadDecisionHistory(); loadFeed(); }) .catch(function (e) { @@ -2749,16 +2927,14 @@ function amountBracketSelectHtml(tx) { }); return ''; } -function assetTypeSelectHtml(tx) { +function assetTypeInputHtml(tx, chamber) { + ensureReviewAssetTypeDatalists(); var current = String(tx.assetType || '').trim(); - var seen = {}; - var opts = ''; - REVIEW_ASSET_TYPES.forEach(function (pair) { - seen[pair[0]] = true; - opts += ''; - }); - if (current && !seen[current]) opts += ''; - return ''; + var category = reviewAssetTypeCategoryLabel(current); + return '' + + '' + + '' + esc(category) + '' + + ''; } function parseBracketValue(v) { var s = String(v || ''); @@ -2766,14 +2942,18 @@ function parseBracketValue(v) { var p = s.split(':'); return { min: p[0] === '' ? null : Number(p[0]), max: p[1] === '' ? null : Number(p[1]) }; } -function syncReviewOptionFlag(selectEl) { - var row = selectEl && selectEl.closest ? selectEl.closest('.me-row') : null; +function syncReviewAssetTypeInput(inputEl) { + var row = inputEl && inputEl.closest ? inputEl.closest('.me-row') : null; var cb = row && row.querySelector ? row.querySelector('.me-option') : null; - if (cb && selectEl.value === 'OP') cb.checked = true; + var value = inputEl ? String(inputEl.value || '').trim() : ''; + var category = reviewAssetTypeCategoryLabel(value); + if (cb && (value.toUpperCase() === 'OP' || category === 'Options')) cb.checked = true; + var label = row && row.querySelector ? row.querySelector('.me-asset-type-category') : null; + if (label) label.textContent = category || ''; } /* Shared review editor. It can start blank for manual entry, from the queued review payload, or from any selected model run. Submit stays explicit. */ -function meRowHtml(tx) { +function meRowHtml(tx, chamber) { tx = normalizeReviewEdit(tx || {}, 'review editor'); return '
' + ' ' + @@ -2781,7 +2961,7 @@ function meRowHtml(tx) { amountBracketSelectHtml(tx) + ' ' + ' ' + - assetTypeSelectHtml(tx) + + assetTypeInputHtml(tx, chamber) + '' + '' + '' + @@ -2791,21 +2971,33 @@ function meRowHtml(tx) { '' + '
'; } -function meAddRow(docId, tx) { var c = el('me-rows-' + docId); if (c) c.insertAdjacentHTML('beforeend', meRowHtml(tx)); } +function meAddRow(docId, tx) { + var c = el('me-rows-' + docId); + var tr = el('me-' + docId); + var chamber = tr ? tr.getAttribute('data-chamber') : ''; + if (c) c.insertAdjacentHTML('beforeend', meRowHtml(tx, chamber)); +} function meCancel(docId) { var tr = el('me-' + docId); if (tr) tr.parentNode.removeChild(tr); } +function reviewItemForDoc(docId) { + for (var i = 0; i < REVIEW.length; i++) { if (REVIEW[i].docId === docId) return REVIEW[i]; } + return null; +} function openQueuedReviewEditor(docId) { - var item = null; - for (var i = 0; i < REVIEW.length; i++) { if (REVIEW[i].docId === docId) { item = REVIEW[i]; break; } } + var item = reviewItemForDoc(docId); var rows = reviewPayloadTransactions(item && item.payload); - openReviewEditor(docId, rows, 'confirm', 'queued extracted rows'); + openReviewEditor(docId, rows, 'confirm', 'queued extracted rows', item && item.chamber); } function useModelRows(docId, idx) { var run = REVIEW_RUNS[docId] && REVIEW_RUNS[docId][idx]; if (!run || !run.rows || !run.rows.length) { alert('That model run has no rows to use.'); return; } - openReviewEditor(docId, run.rows, 'confirm', run.provider + ':' + run.model); + var item = reviewItemForDoc(docId); + openReviewEditor(docId, run.rows, 'confirm', run.provider + ':' + run.model, item && item.chamber); +} +function manualEntry(docId) { + var item = reviewItemForDoc(docId); + openReviewEditor(docId, [], 'manual', 'manual entry', item && item.chamber); } -function manualEntry(docId) { openReviewEditor(docId, [], 'manual', 'manual entry'); } -function openReviewEditor(docId, rows, decision, label) { +function openReviewEditor(docId, rows, decision, label, chamber) { var old = el('me-' + docId); if (old && old.parentNode) old.parentNode.removeChild(old); var row = el('rv-' + docId); @@ -2813,6 +3005,7 @@ function openReviewEditor(docId, rows, decision, label) { var tr = document.createElement('tr'); tr.id = 'me-' + docId; tr.setAttribute('data-decision', decision); + tr.setAttribute('data-chamber', chamber || ''); var safeLabel = label || (decision === 'manual' ? 'manual entry' : 'selected rows'); var title = decision === 'manual' ? 'Manual Entry' : 'Edit Rows To Confirm'; var submit = decision === 'manual' ? 'Submit Manual Entry' : 'Confirm Edited Rows'; @@ -2844,7 +3037,7 @@ function meSubmit(docId) { if (!t && !asset) return; // skip blank rows var bracket = parseBracketValue(g.querySelector('.me-bracket').value); var conf = g.querySelector('.me-conf').value; - var assetType = (g.querySelector('.me-asset-type').value || '').trim(); + var assetType = reviewNormalizeAssetTypeValue(g.querySelector('.me-asset-type').value || ''); edits.push({ ticker: t || null, assetName: asset || t || '(review entry)', @@ -2854,7 +3047,8 @@ function meSubmit(docId) { txDate: g.querySelector('.me-date').value || null, owner: g.querySelector('.me-owner').value, assetType: assetType || null, - isOption: g.querySelector('.me-option').checked || assetType === 'OP', + assetTypeName: reviewAssetTypeName(assetType) || null, + isOption: g.querySelector('.me-option').checked || assetType === 'OP' || reviewAssetTypeCategoryLabel(assetType) === 'Options', capGainsOver200: g.querySelector('.me-cap').checked, rawText: (g.querySelector('.me-raw').value || '').trim() || (decision === 'manual' ? 'manual entry' : 'review editor'), confidence: conf === '' ? (decision === 'manual' ? 1 : null) : Number(conf) @@ -2867,7 +3061,7 @@ function meSubmit(docId) { body: JSON.stringify({ decision: decision, edits: edits }) }) .then(okOrThrow) - .then(function () { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); if (tr && tr.parentNode) tr.parentNode.removeChild(tr); renderReview(); loadFeed(); }) + .then(function () { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); if (tr && tr.parentNode) tr.parentNode.removeChild(tr); renderReview(); loadDecisionHistory(); loadFeed(); }) .catch(function (e) { if (tr) tr.querySelectorAll('button,input,select').forEach(function (b) { b.disabled = false; }); alert(isAuthError(e) ? ADMIN_MOVED_MSG : ('Review submit failed: ' + e.message));