From 744e998a006999edd9d48ef5a1f6b3d699e51457 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:29:08 -0500 Subject: [PATCH 01/15] feat: canonicalize disclosure asset types --- app/docs/client-mobile-api.md | 6 +- app/docs/fmp-data-sharing.md | 3 +- app/docs/pit-score-export.md | 5 +- app/src/analytics/__tests__/builders.test.ts | 8 +- app/src/analytics/builders.ts | 9 +- app/src/analytics/routes.ts | 13 +- app/src/client/__tests__/routes.test.ts | 29 +- app/src/client/routes.ts | 8 + app/src/delivery/rows.ts | 10 +- app/src/export/__tests__/pitScores.test.ts | 10 + app/src/export/pitScores.ts | 61 ++- .../extraction/__tests__/senateHtml.test.ts | 2 + app/src/extraction/normalizer.ts | 7 + app/src/extraction/senateHtml.ts | 4 +- app/src/extraction/textPdf.ts | 61 +-- app/src/shared/__tests__/assetTypes.test.ts | 57 +++ app/src/shared/assetTypes.ts | 373 ++++++++++++++++++ app/src/shared/types.ts | 11 + 18 files changed, 583 insertions(+), 94 deletions(-) create mode 100644 app/src/shared/__tests__/assetTypes.test.ts create mode 100644 app/src/shared/assetTypes.ts diff --git a/app/docs/client-mobile-api.md b/app/docs/client-mobile-api.md index adf0b0190..676b5828f 100644 --- a/app/docs/client-mobile-api.md +++ b/app/docs/client-mobile-api.md @@ -60,8 +60,10 @@ shared type set but still return `501`. `chamber`, `type`, `from`, `to`, `order`, and `limit`, and returns the cursor/count/total metadata used by polling clients. - Each feed item's `asset` object carries `name` (the disclosed asset text), - `ticker`, `type`, `sector`, and `marketCapBucket`, plus two enrichment fields - shared with the web client so every surface renders identically: + `ticker`, raw disclosure `type`, `typeName`, canonical cross-chamber + `typeCategory` / `typeCategoryLabel`, `sector`, and `marketCapBucket`, plus + two enrichment fields shared with the web client so every surface renders + identically: - `companyName` — the canonical company name from `securities_ref` (`null` until the ticker is enriched). - `logoUrl` — a same-origin path to the cached logo proxy, e.g. diff --git a/app/docs/fmp-data-sharing.md b/app/docs/fmp-data-sharing.md index e9be71b82..91a45be80 100644 --- a/app/docs/fmp-data-sharing.md +++ b/app/docs/fmp-data-sharing.md @@ -236,7 +236,8 @@ feed rows or public analytics. Per-transaction object (each item in `transactions[]`): ``` -{ id, docId, filerId, txDate, owner, assetName, ticker, assetType, txType, +{ id, docId, filerId, txDate, owner, assetName, ticker, assetType, + assetTypeName, assetTypeCategory, assetTypeCategoryLabel, txType, amountMin, amountMax, isOption, capGainsOver200, rawText, confidence, source, createdAt, cursorSeq, fullName, state, photoUrl, filedDate, firstSeenAt, diff --git a/app/docs/pit-score-export.md b/app/docs/pit-score-export.md index d6c521d85..b8dd0eb95 100644 --- a/app/docs/pit-score-export.md +++ b/app/docs/pit-score-export.md @@ -26,7 +26,10 @@ whether returned rows are safe for historical validation. Rows are keyed by ticker and market-available disclosure timestamp: - `observationId` -- `ticker`, `stableSecurityId`, `cusip`, `cik`, `assetType` +- `ticker`, `stableSecurityId`, `cusip`, `cik` +- raw/security asset context: `assetType`, `assetTypeName` +- cross-chamber canonical asset context: `assetTypeCategory`, + `assetTypeCategoryLabel`, `assetTypeCategorySource` - `tickerMapVersion`, `delistingTickerChangeMetadata` - `asOf`, `disclosureAvailableAt`, `computedAt`, `dataCutoffAt` - `scoreVersion`, `parameterManifest` diff --git a/app/src/analytics/__tests__/builders.test.ts b/app/src/analytics/__tests__/builders.test.ts index 21f2ed996..4e7fdec2c 100644 --- a/app/src/analytics/__tests__/builders.test.ts +++ b/app/src/analytics/__tests__/builders.test.ts @@ -173,10 +173,12 @@ describe('buildPartySplitQuery', () => { }); describe('buildSectorBreakdownQuery', () => { - it('buckets the asset_type and coalesces empties to Unknown', () => { + it('buckets canonical asset-type categories across raw source labels', () => { const q = buildSectorBreakdownQuery({ window: 'all' }); - expect(q.sql).toContain("COALESCE(NULLIF(t.asset_type, ''), 'Unknown') AS asset_type"); - expect(q.sql).toContain('GROUP BY asset_type'); + expect(q.sql).toContain('AS asset_type_category'); + expect(q.sql).toContain("THEN 'public_equity'"); + expect(q.sql).toContain("THEN 'fixed_income_government'"); + expect(q.sql).toContain('GROUP BY asset_type_category'); }); }); diff --git a/app/src/analytics/builders.ts b/app/src/analytics/builders.ts index 4b0b2b4f8..399718f06 100644 --- a/app/src/analytics/builders.ts +++ b/app/src/analytics/builders.ts @@ -15,6 +15,7 @@ */ import type { SqlParam } from '../shared/db'; +import { canonicalAssetTypeCategorySql } from '../shared/assetTypes'; import { ANALYTICS_FROM_JOINS, ANALYTICS_FROM_JOINS_SECURITIES, @@ -321,21 +322,23 @@ export function buildPartySplitOverTimeQuery( } // --------------------------------------------------------------------------- -// 8. Sector breakdown — by asset_type (the only classification we have) +// 8. Instrument-type breakdown - canonicalized across House codes and Senate labels. // --------------------------------------------------------------------------- export function buildSectorBreakdownQuery(p: CommonFilters & { limit?: number }): BuiltQuery { const { where, params } = buildCommonFilters(p); const limit = clampLimit(p.limit, 20, 100); + const categorySql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); const sql = - "SELECT COALESCE(NULLIF(t.asset_type, ''), 'Unknown') AS asset_type, " + + `SELECT ${categorySql} AS asset_type_category, ` + + "GROUP_CONCAT(DISTINCT COALESCE(NULLIF(t.asset_type, ''), 'Unknown')) AS raw_asset_types, " + 'COUNT(*) AS trade_count, ' + `${BUY} AS buy_count, ${SELL} AS sell_count, ` + `SUM(${MID}) AS est_volume, ` + `COUNT(DISTINCT CASE WHEN ${TICKER_RESOLVED_SQL} THEN t.ticker END) AS unique_tickers ` + ANALYTICS_FROM_JOINS + whereSql(where) + - 'GROUP BY asset_type ORDER BY trade_count DESC ' + + 'GROUP BY asset_type_category ORDER BY trade_count DESC ' + `LIMIT ${limit}`; return { sql, params }; } diff --git a/app/src/analytics/routes.ts b/app/src/analytics/routes.ts index a461f26ba..62565a19b 100644 --- a/app/src/analytics/routes.ts +++ b/app/src/analytics/routes.ts @@ -20,6 +20,7 @@ import { Hono } from 'hono'; import type { Env } from '../shared/types'; import { all, get, parseJson } from '../shared/db'; +import { assetTypeCategoryLabel, isAssetTypeCategory } from '../shared/assetTypes'; import { asChamber, asPartyBucket, @@ -95,6 +96,14 @@ function usd(v: unknown): number { function str(v: unknown): string | null { return typeof v === 'string' && v.length > 0 ? v : null; } +function assetTypeCategory(v: unknown) { + const s = str(v); + return isAssetTypeCategory(s) ? s : 'unknown'; +} +function rawList(v: unknown): string[] { + const s = str(v); + return s ? s.split(',').map((part) => part.trim()).filter(Boolean) : []; +} /** Split a list into fixed-size batches (to stay under D1's 100-bound-param cap). */ function chunk(items: T[], size: number): T[][] { const out: T[][] = []; @@ -658,7 +667,9 @@ export function buildAnalyticsRouter(): Hono<{ Bindings: Env }> { const built = buildSectorBreakdownQuery({ ...f, limit }); const rows = await all>(c.env.DB, built.sql, built.params); const sectors = rows.map((row) => ({ - assetType: str(row.asset_type) ?? 'Unknown', + assetType: assetTypeCategoryLabel(assetTypeCategory(row.asset_type_category)), + assetTypeCategory: assetTypeCategory(row.asset_type_category), + rawAssetTypes: rawList(row.raw_asset_types), tradeCount: num(row.trade_count), buyCount: num(row.buy_count), sellCount: num(row.sell_count), diff --git a/app/src/client/__tests__/routes.test.ts b/app/src/client/__tests__/routes.test.ts index ea0522a96..8edcaf6b4 100644 --- a/app/src/client/__tests__/routes.test.ts +++ b/app/src/client/__tests__/routes.test.ts @@ -288,7 +288,14 @@ describe('client API routes', () => { expect(res.status).toBe(200); const body = (await res.json()) as { items: Array<{ - asset: { ticker: string | null; companyName: string | null; logoUrl: string | null; sector: string | null }; + asset: { + ticker: string | null; + companyName: string | null; + logoUrl: string | null; + sector: string | null; + typeCategory: string; + typeCategoryLabel: string; + }; }>; }; expect(body.items[0].asset).toMatchObject({ @@ -296,6 +303,8 @@ describe('client API routes', () => { companyName: 'Apple Inc.', logoUrl: '/api/logos/ticker?symbol=AAPL', sector: 'Technology', + typeCategory: 'public_equity', + typeCategoryLabel: 'Public Equity', }); }); @@ -336,9 +345,23 @@ describe('client API routes', () => { const res = await app.request('http://localhost/feed?limit=1', {}, env); expect(res.status).toBe(200); const body = (await res.json()) as { - items: Array<{ asset: { ticker: string | null; companyName: string | null; logoUrl: string | null } }>; + items: Array<{ + asset: { + ticker: string | null; + companyName: string | null; + logoUrl: string | null; + typeCategory: string; + typeCategoryLabel: string; + }; + }>; }; - expect(body.items[0].asset).toMatchObject({ ticker: null, companyName: null, logoUrl: null }); + expect(body.items[0].asset).toMatchObject({ + ticker: null, + companyName: null, + logoUrl: null, + typeCategory: 'fixed_income_government', + typeCategoryLabel: 'Government / Municipal Debt', + }); }); it('updates preferences through an authenticated command', async () => { diff --git a/app/src/client/routes.ts b/app/src/client/routes.ts index 69fe6ee9b..11a2efdcf 100644 --- a/app/src/client/routes.ts +++ b/app/src/client/routes.ts @@ -28,6 +28,7 @@ import { type FeedTransactionRow, type TxQueryParams, } from '../delivery/rows'; +import { canonicalizeAssetType } from '../shared/assetTypes'; import { createSubscription, getSubscription, @@ -203,6 +204,10 @@ function clientLogoUrl(ticker: string | null): string | null { function clientTradeFromRow(row: FeedTransactionRow & { __chamber?: string | null; __member_name?: string | null; __party?: string | null }): ClientTrade { const tx = mapFeedTransaction(row); + const assetType = canonicalizeAssetType(tx.assetType, tx.assetTypeName, { + isOption: tx.isOption, + assetName: tx.assetName, + }); return { id: tx.id, cursor: tx.cursorSeq, @@ -221,6 +226,9 @@ function clientTradeFromRow(row: FeedTransactionRow & { __chamber?: string | nul companyName: tx.refCompanyName ?? null, logoUrl: clientLogoUrl(tx.ticker), type: tx.assetType, + typeName: tx.assetTypeName ?? null, + typeCategory: assetType.category, + typeCategoryLabel: assetType.categoryLabel, sector: tx.refSector ?? null, marketCapBucket: tx.refMarketCapBucket ?? null, }, diff --git a/app/src/delivery/rows.ts b/app/src/delivery/rows.ts index 40e7d7670..15b8e00df 100644 --- a/app/src/delivery/rows.ts +++ b/app/src/delivery/rows.ts @@ -20,6 +20,7 @@ import type { TxType, } from '../shared/types'; import { parseJson, toBool } from '../shared/db'; +import { canonicalizeAssetType } from '../shared/assetTypes'; // --------------------------------------------------------------------------- // Raw row shapes (mirror the D1 column names in migrations/0001_init.sql) @@ -112,6 +113,11 @@ export interface FilingRow { // --------------------------------------------------------------------------- export function mapTransaction(row: TransactionRow): Transaction { + const isOption = toBool(row.is_option); + const assetType = canonicalizeAssetType(row.asset_type, row.asset_type_name ?? null, { + isOption, + assetName: row.asset_name ?? null, + }); return { id: row.id, docId: row.doc_id, @@ -122,10 +128,12 @@ export function mapTransaction(row: TransactionRow): Transaction { ticker: row.ticker, assetType: row.asset_type, assetTypeName: row.asset_type_name ?? null, + assetTypeCategory: assetType.category, + assetTypeCategoryLabel: assetType.categoryLabel, txType: (row.tx_type as TxType) ?? 'P', amountMin: row.amount_min, amountMax: row.amount_max, - isOption: toBool(row.is_option), + isOption, capGainsOver200: toBool(row.cap_gains_over_200), rawText: row.raw_text ?? '', filingStatus: row.filing_status ?? null, diff --git a/app/src/export/__tests__/pitScores.test.ts b/app/src/export/__tests__/pitScores.test.ts index 2d3a5b3cb..fe24e4846 100644 --- a/app/src/export/__tests__/pitScores.test.ts +++ b/app/src/export/__tests__/pitScores.test.ts @@ -144,6 +144,16 @@ describe('buildPitScoreExport pagination', () => { const first = await buildPitScoreExport(env as never, { limit: 1, format: 'json', placebo: 'none', source: 'all' }, new Date('2026-03-01T00:00:00.000Z')); expect(first.rows.map((r) => r.ticker)).toEqual(['AAPL']); expect(first.pagination.nextCursor).toBe('2026-01-01T00:00:00.000Z~AAPL'); + expect(first.rows[0]).toMatchObject({ + assetTypeCategory: 'public_equity', + assetTypeCategoryLabel: 'Public Equity', + assetTypeCategorySource: 'label', + }); + expect(first.rows[0].includedDisclosures[0]).toMatchObject({ + assetType: 'STOCK', + assetTypeName: 'Stock', + assetTypeCategory: 'public_equity', + }); expect(first.validationReadiness).toMatchObject({ historicalValidationReady: false, scoreInputsPitSafeRows: 1, diff --git a/app/src/export/pitScores.ts b/app/src/export/pitScores.ts index e45e51bf4..ace88469f 100644 --- a/app/src/export/pitScores.ts +++ b/app/src/export/pitScores.ts @@ -14,6 +14,7 @@ import { bracketMidpoint, netSentiment, round } from '../analytics/compute'; import { committeeConflict } from '../analytics/conflicts'; import { TICKER_ALIASES } from '../extraction/tickerNormalize'; import { pctChange } from '../prices/compute'; +import { canonicalizeAssetType } from '../shared/assetTypes'; export const PIT_SCORE_VERSION = 'congress-pit-v2'; export const TICKER_MAP_VERSION = 'ticker-normalize-v1'; @@ -158,6 +159,10 @@ interface PitScoreRow { cusip: string | null; cik: string | null; assetType: string | null; + assetTypeName: string | null; + assetTypeCategory: string; + assetTypeCategoryLabel: string; + assetTypeCategorySource: string; tickerMapVersion: string; delistingTickerChangeMetadata: Record; asOf: string; @@ -1197,25 +1202,39 @@ async function buildRow( const signedScore = score == null ? null : direction === 'SELL' ? -score : direction === 'BUY' ? score : 0; const price = await (priceCache.get(ticker) ?? priceCache.set(ticker, priceSeries(env, ticker)).get(ticker)!); const ref = txs.find((t) => t.company_name || t.cik || t.asset_class || t.sector) ?? txs[0]; - const includedDisclosures = txs.map((t) => ({ - availabilitySource: availabilityFor(t).source, - availabilityPrecision: availabilityFor(t).precision, - disclosureId: t.id, - docId: t.doc_id, - sourceUrl: t.source_url, - hashedFilerId: stableFilerHash(t.filer_id), - txDate: t.tx_date, - disclosedAt: isoTimestamp(t.first_seen_at) ?? isoTimestamp(t.filed_date) ?? t.created_at, - filedAt: t.filed_date, - side: t.tx_type, - owner: t.owner, - amountLow: finiteOrNull(t.amount_min), - amountHigh: finiteOrNull(t.amount_max), - amountEstimate: Math.round(midpoint(t)), - chamber: t.filer_chamber ?? t.filing_chamber, - amendmentFlag: false, - cancelFlag: false, - })); + const canonicalAssetType = canonicalizeAssetType(ref?.asset_type ?? null, ref?.asset_type_name ?? null, { + isOption: txs.some((t) => t.is_option === 1), + assetName: ref?.asset_name ?? null, + }); + const includedDisclosures = txs.map((t) => { + const disclosureAssetType = canonicalizeAssetType(t.asset_type, t.asset_type_name, { + isOption: t.is_option === 1, + assetName: t.asset_name, + }); + return { + availabilitySource: availabilityFor(t).source, + availabilityPrecision: availabilityFor(t).precision, + disclosureId: t.id, + docId: t.doc_id, + sourceUrl: t.source_url, + hashedFilerId: stableFilerHash(t.filer_id), + txDate: t.tx_date, + disclosedAt: isoTimestamp(t.first_seen_at) ?? isoTimestamp(t.filed_date) ?? t.created_at, + filedAt: t.filed_date, + side: t.tx_type, + owner: t.owner, + amountLow: finiteOrNull(t.amount_min), + amountHigh: finiteOrNull(t.amount_max), + amountEstimate: Math.round(midpoint(t)), + chamber: t.filer_chamber ?? t.filing_chamber, + assetType: t.asset_type, + assetTypeName: t.asset_type_name, + assetTypeCategory: disclosureAssetType.category, + assetTypeCategoryLabel: disclosureAssetType.categoryLabel, + amendmentFlag: false, + cancelFlag: false, + }; + }); const aliasesFrom = Object.entries(TICKER_ALIASES).filter(([, to]) => to === ticker).map(([from]) => from); const clusterConsensus = buildClusterConsensus(ticker, asOf, txs, allTxRows, memberSkill); const pitValidity = buildPitValidity(txs); @@ -1226,6 +1245,10 @@ async function buildRow( cusip: null, cik: ref?.cik ?? null, assetType: ref?.asset_class ?? ref?.asset_type ?? null, + assetTypeName: ref?.asset_type_name ?? null, + assetTypeCategory: canonicalAssetType.category, + assetTypeCategoryLabel: canonicalAssetType.categoryLabel, + assetTypeCategorySource: canonicalAssetType.source, tickerMapVersion: TICKER_MAP_VERSION, delistingTickerChangeMetadata: { knownPriorTickers: aliasesFrom, diff --git a/app/src/extraction/__tests__/senateHtml.test.ts b/app/src/extraction/__tests__/senateHtml.test.ts index ab718e235..90eb76a73 100644 --- a/app/src/extraction/__tests__/senateHtml.test.ts +++ b/app/src/extraction/__tests__/senateHtml.test.ts @@ -75,6 +75,8 @@ describe('SenateHtmlExtractor', () => { expect(r1.owner).toBe('spouse'); expect(r1.ticker).toBe('AAPL'); expect(r1.assetName).toContain('Apple'); + expect(r1.assetType).toBe('Stock'); + expect(r1.assetTypeName).toBe('Stock'); expect(r1.txType).toBe('P'); expect(r1.amountMin).toBe(1001); expect(r1.amountMax).toBe(15000); diff --git a/app/src/extraction/normalizer.ts b/app/src/extraction/normalizer.ts index 920ed71ed..e28e89091 100644 --- a/app/src/extraction/normalizer.ts +++ b/app/src/extraction/normalizer.ts @@ -21,6 +21,7 @@ import type { Env, Filing, Owner, ParsedTx, Transaction, TxType } from '../shared/types'; import { all, run, fromBool, parseJson } from '../shared/db'; import { isValidBracket, matchBracket, nearestBracket } from '../shared/brackets'; +import { canonicalizeAssetType } from '../shared/assetTypes'; import { uuid } from '../shared/ids'; import { isPlaceholderTicker, resolveTickerDeterministic, TICKER_ALIASES } from './tickerNormalize'; @@ -238,6 +239,10 @@ function buildTransaction( filing.filedDate, resolve, ); + const assetType = canonicalizeAssetType(p.assetType, p.assetTypeName ?? null, { + isOption: p.isOption, + assetName: p.assetName, + }); const tx: Transaction = { id: uuid(), @@ -249,6 +254,8 @@ function buildTransaction( ticker: s.ticker, assetType: p.assetType, assetTypeName: p.assetTypeName ?? null, + assetTypeCategory: assetType.category, + assetTypeCategoryLabel: assetType.categoryLabel, txType: s.txType, amountMin: s.amountMin, amountMax: s.amountMax, diff --git a/app/src/extraction/senateHtml.ts b/app/src/extraction/senateHtml.ts index 4f1069caf..d7c310ffa 100644 --- a/app/src/extraction/senateHtml.ts +++ b/app/src/extraction/senateHtml.ts @@ -231,13 +231,15 @@ function rowToParsedTx(cells: string[], map: ColumnMap, confidence: number): Par const ticker = normalizeTicker(tickerRaw); const combinedText = cells.join(' '); const capGainsCell = get(map.capGains); + const assetType = get(map.assetType).trim() || null; return { txDate: normalizeDate(get(map.date)), owner: normalizeOwner(get(map.owner), assetName), assetName: assetName || tickerRaw || '(unknown)', ticker, - assetType: get(map.assetType).trim() || null, + assetType, + assetTypeName: assetType, txType, amountMin: min, amountMax: max, diff --git a/app/src/extraction/textPdf.ts b/app/src/extraction/textPdf.ts index ae401e425..2a8a60f79 100644 --- a/app/src/extraction/textPdf.ts +++ b/app/src/extraction/textPdf.ts @@ -20,6 +20,7 @@ import { extractText, getDocumentProxy } from 'unpdf'; import type { Extractor, ExtractorInput, ExtractorResult } from '../extractors/types'; import type { Filing, Owner, ParsedTx, TxType } from '../shared/types'; +import { HOUSE_ASSET_TYPE_NAMES, houseAssetTypeCodePattern } from '../shared/assetTypes'; import { parseAmountRange } from './amounts'; import { detectOption } from './senateHtml'; @@ -80,62 +81,8 @@ const OWNER_CODES: Record = { SELF: 'self', }; -const HOUSE_ASSET_TYPE_NAMES: Record = { - '4K': '401K and Other Non-Federal Retirement Accounts', - '5C': '529 College Savings Plan', - '5F': '529 Portfolio', - '5P': '529 Prepaid Tuition Plan', - AB: 'Asset-Backed Securities', - BA: 'Bank Accounts, Money Market Accounts and CDs', - BK: 'Brokerage Accounts', - CO: 'Collectibles', - CS: 'Corporate Securities (Bonds and Notes)', - CT: 'Cryptocurrency', - DB: 'Defined Benefit Pension', - DO: 'Debts Owed to the Filer', - DS: 'Delaware Statutory Trust', - EF: 'Exchange Traded Funds (ETF)', - EQ: 'Excepted/Qualified Blind Trust', - ET: 'Exchange Traded Notes', - FA: 'Farms', - FE: 'Foreign Exchange Position (Currency)', - FN: 'Fixed Annuity', - FU: 'Futures', - GS: 'Government Securities and Agency Debt', - HE: 'Hedge Funds & Private Equity Funds (EIF)', - HN: 'Hedge Funds & Private Equity Funds (non-EIF)', - IC: 'Investment Club', - IH: 'IRA (Held in Cash)', - IP: 'Intellectual Property & Royalties', - IR: 'IRA', - MA: 'Managed Accounts (e.g., SMA and UMA)', - MF: 'Mutual Funds', - MO: 'Mineral/Oil/Solar Energy Rights', - OI: 'Ownership Interest (Holding Investments)', - OL: 'Ownership Interest (Engaged in a Trade or Business)', - OP: 'Options', - OT: 'Other', - PE: 'Pensions', - PM: 'Precious Metals', - PS: 'Stock (Not Publicly Traded)', - RE: 'Real Estate Invest. Trust (REIT)', - RF: 'REIT (EIF)', - RN: 'REIT (non-EIF)', - RP: 'Real Property', - RS: 'Restricted Stock Units (RSUs)', - SA: 'Stock Appreciation Right', - ST: 'Stocks (including ADRs)', - TR: 'Trust', - VA: 'Variable Annuity', - VI: 'Variable Insurance', - WU: 'Whole/Universal Insurance', -}; - // Asset-type bracket codes used by the House template, e.g. [ST] [OP] [GS] [4K]. -const HOUSE_ASSET_TYPE_CODE_PATTERN = Object.keys(HOUSE_ASSET_TYPE_NAMES) - .sort((a, b) => b.length - a.length) - .map(escapeRegExp) - .join('|'); +const HOUSE_ASSET_TYPE_CODE_PATTERN = houseAssetTypeCodePattern(); const ASSET_TYPE_RE = new RegExp(`\\[(${HOUSE_ASSET_TYPE_CODE_PATTERN})\\]`, 'i'); // A date in MM/DD/YYYY. const DATE_RE = /\b(\d{1,2}\/\d{1,2}\/\d{2,4})\b/g; @@ -275,10 +222,6 @@ function stripLeadingOwnerCode(value: string): string { return value.replace(/^(SP|DC|JT|SELF)\b\s*/i, '').trim(); } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - function normalizeTicker(value: string | null): string | null { if (!value) return null; const ticker = value.toUpperCase().replace('/', '.'); diff --git a/app/src/shared/__tests__/assetTypes.test.ts b/app/src/shared/__tests__/assetTypes.test.ts new file mode 100644 index 000000000..37e6054aa --- /dev/null +++ b/app/src/shared/__tests__/assetTypes.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'vitest'; +import { + assetTypeCategoryLabel, + canonicalAssetTypeCategorySql, + canonicalizeAssetType, + houseAssetTypeCodePattern, +} from '../assetTypes'; + +describe('canonicalizeAssetType', () => { + it('maps House codes and Senate labels into the same public equity category', () => { + expect(canonicalizeAssetType('ST').category).toBe('public_equity'); + expect(canonicalizeAssetType('Stock').category).toBe('public_equity'); + expect(canonicalizeAssetType('ST').label).toBe('Stocks (including ADRs)'); + }); + + it('maps government and municipal debt across chambers', () => { + expect(canonicalizeAssetType('GS').category).toBe('fixed_income_government'); + expect(canonicalizeAssetType('Municipal Security').category).toBe('fixed_income_government'); + expect(canonicalizeAssetType('GS').categoryLabel).toBe('Government / Municipal Debt'); + }); + + it('recognizes digit-prefixed retirement and 529 House codes', () => { + for (const code of ['4K', '5C', '5F', '5P']) { + expect(canonicalizeAssetType(code).category).toBe('retirement_or_529'); + } + }); + + it('keeps private/non-public equity distinct from public stock', () => { + expect(canonicalizeAssetType('PS').category).toBe('private_equity'); + expect(canonicalizeAssetType('Non-Public Stock').category).toBe('private_equity'); + }); + + it('falls back from explicit option flag when the raw type is missing', () => { + expect(canonicalizeAssetType(null, null, { isOption: true }).category).toBe('option'); + }); + + it('treats blank and PDF placeholder rows as unknown', () => { + expect(canonicalizeAssetType(null).category).toBe('unknown'); + expect(canonicalizeAssetType('PDF Disclosed Filing').category).toBe('unknown'); + expect(assetTypeCategoryLabel('unknown')).toBe('Unknown'); + }); + + it('exposes a House-code pattern that includes digit-prefixed codes', () => { + const pattern = houseAssetTypeCodePattern(); + expect(new RegExp(`^(?:${pattern})$`).test('4K')).toBe(true); + expect(new RegExp(`^(?:${pattern})$`).test('GS')).toBe(true); + }); +}); + +describe('canonicalAssetTypeCategorySql', () => { + it('generates a SQL CASE expression for grouped analytics', () => { + const sql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); + expect(sql).toContain("THEN 'public_equity'"); + expect(sql).toContain("THEN 'fixed_income_government'"); + expect(sql).toContain('t.is_option = 1'); + }); +}); diff --git a/app/src/shared/assetTypes.ts b/app/src/shared/assetTypes.ts new file mode 100644 index 000000000..088dbbb6d --- /dev/null +++ b/app/src/shared/assetTypes.ts @@ -0,0 +1,373 @@ +/** + * Shared instrument-type canonicalization. + * + * Keep raw disclosure values in transactions.asset_type / asset_type_name, then + * compute this layer at read/export time so House bracket codes and Senate eFD + * labels can roll up together without erasing provenance. + */ + +export type AssetTypeCategory = + | 'public_equity' + | 'private_equity' + | 'option' + | 'fund' + | 'fixed_income_government' + | 'fixed_income_corporate' + | 'fixed_income_asset_backed' + | 'cash' + | 'retirement_or_529' + | 'real_estate' + | 'private_fund' + | 'business_interest' + | 'crypto' + | 'insurance_annuity' + | 'trust' + | 'commodity_collectible' + | 'derivative' + | 'intellectual_property' + | 'receivable' + | 'other_security' + | 'other' + | 'unknown'; + +export type AssetTypeSource = 'house_code' | 'label' | 'option_flag' | 'missing' | 'unknown'; + +export interface CanonicalAssetType { + rawType: string | null; + rawName: string | null; + code: string | null; + label: string; + category: AssetTypeCategory; + categoryLabel: string; + source: AssetTypeSource; +} + +export const HOUSE_ASSET_TYPE_NAMES: Record = { + '4K': '401K and Other Non-Federal Retirement Accounts', + '5C': '529 College Savings Plan', + '5F': '529 Portfolio', + '5P': '529 Prepaid Tuition Plan', + AB: 'Asset-Backed Securities', + BA: 'Bank Accounts, Money Market Accounts and CDs', + BK: 'Brokerage Accounts', + CO: 'Collectibles', + CS: 'Corporate Securities (Bonds and Notes)', + CT: 'Cryptocurrency', + DB: 'Defined Benefit Pension', + DO: 'Debts Owed to the Filer', + DS: 'Delaware Statutory Trust', + EF: 'Exchange Traded Funds (ETF)', + EQ: 'Excepted/Qualified Blind Trust', + ET: 'Exchange Traded Notes', + FA: 'Farms', + FE: 'Foreign Exchange Position (Currency)', + FN: 'Fixed Annuity', + FU: 'Futures', + GS: 'Government Securities and Agency Debt', + HE: 'Hedge Funds & Private Equity Funds (EIF)', + HN: 'Hedge Funds & Private Equity Funds (non-EIF)', + IC: 'Investment Club', + IH: 'IRA (Held in Cash)', + IP: 'Intellectual Property & Royalties', + IR: 'IRA', + MA: 'Managed Accounts (e.g., SMA and UMA)', + MF: 'Mutual Funds', + MO: 'Mineral/Oil/Solar Energy Rights', + OI: 'Ownership Interest (Holding Investments)', + OL: 'Ownership Interest (Engaged in a Trade or Business)', + OP: 'Options', + OT: 'Other', + PE: 'Pensions', + PM: 'Precious Metals', + PS: 'Stock (Not Publicly Traded)', + RE: 'Real Estate Invest. Trust (REIT)', + RF: 'REIT (EIF)', + RN: 'REIT (non-EIF)', + RP: 'Real Property', + RS: 'Restricted Stock Units (RSUs)', + SA: 'Stock Appreciation Right', + ST: 'Stocks (including ADRs)', + TR: 'Trust', + VA: 'Variable Annuity', + VI: 'Variable Insurance', + WU: 'Whole/Universal Insurance', +}; + +export const HOUSE_ASSET_TYPE_CATEGORIES: Record = { + '4K': 'retirement_or_529', + '5C': 'retirement_or_529', + '5F': 'retirement_or_529', + '5P': 'retirement_or_529', + AB: 'fixed_income_asset_backed', + BA: 'cash', + BK: 'cash', + CO: 'commodity_collectible', + CS: 'fixed_income_corporate', + CT: 'crypto', + DB: 'retirement_or_529', + DO: 'receivable', + DS: 'trust', + EF: 'fund', + EQ: 'trust', + ET: 'fund', + FA: 'commodity_collectible', + FE: 'derivative', + FN: 'insurance_annuity', + FU: 'derivative', + GS: 'fixed_income_government', + HE: 'private_fund', + HN: 'private_fund', + IC: 'business_interest', + IH: 'retirement_or_529', + IP: 'intellectual_property', + IR: 'retirement_or_529', + MA: 'fund', + MF: 'fund', + MO: 'commodity_collectible', + OI: 'business_interest', + OL: 'business_interest', + OP: 'option', + OT: 'other', + PE: 'retirement_or_529', + PM: 'commodity_collectible', + PS: 'private_equity', + RE: 'real_estate', + RF: 'real_estate', + RN: 'real_estate', + RP: 'real_estate', + RS: 'derivative', + SA: 'derivative', + ST: 'public_equity', + TR: 'trust', + VA: 'insurance_annuity', + VI: 'insurance_annuity', + WU: 'insurance_annuity', +}; + +export const ASSET_TYPE_CATEGORY_LABELS: Record = { + public_equity: 'Public Equity', + private_equity: 'Private Equity', + option: 'Options', + fund: 'Funds / ETFs / REITs', + fixed_income_government: 'Government / Municipal Debt', + fixed_income_corporate: 'Corporate Debt', + fixed_income_asset_backed: 'Asset-Backed Securities', + cash: 'Cash / Bank Accounts', + retirement_or_529: 'Retirement / 529 Accounts', + real_estate: 'Real Estate', + private_fund: 'Private Funds', + business_interest: 'Business Interests', + crypto: 'Crypto', + insurance_annuity: 'Insurance / Annuities', + trust: 'Trusts', + commodity_collectible: 'Commodities / Collectibles', + derivative: 'Derivatives / Rights', + intellectual_property: 'Intellectual Property', + receivable: 'Receivables', + other_security: 'Other Securities', + other: 'Other', + unknown: 'Unknown', +}; + +const LABEL_CATEGORY_ALIASES: Record = { + '401k and other non federal retirement accounts': 'retirement_or_529', + '401k and other non-federal retirement accounts': 'retirement_or_529', + '529 college savings plan': 'retirement_or_529', + '529 portfolio': 'retirement_or_529', + '529 prepaid tuition plan': 'retirement_or_529', + 'asset backed securities': 'fixed_income_asset_backed', + 'asset-backed securities': 'fixed_income_asset_backed', + 'bank accounts money market accounts and cds': 'cash', + 'brokerage accounts': 'cash', + cash: 'cash', + cd: 'cash', + cds: 'cash', + collectibles: 'commodity_collectible', + 'corporate bond': 'fixed_income_corporate', + 'corporate bonds': 'fixed_income_corporate', + 'corporate debt': 'fixed_income_corporate', + 'corporate securities bonds and notes': 'fixed_income_corporate', + cryptocurrency: 'crypto', + crypto: 'crypto', + 'defined benefit pension': 'retirement_or_529', + 'debts owed to the filer': 'receivable', + 'delaware statutory trust': 'trust', + 'exchange traded funds etf': 'fund', + etf: 'fund', + 'exchange traded notes': 'fund', + etn: 'fund', + 'excepted qualified blind trust': 'trust', + farms: 'commodity_collectible', + 'foreign exchange position currency': 'derivative', + 'fixed annuity': 'insurance_annuity', + futures: 'derivative', + 'government securities and agency debt': 'fixed_income_government', + 'municipal security': 'fixed_income_government', + muni: 'fixed_income_government', + 'government municipal debt': 'fixed_income_government', + 'hedge funds private equity funds eif': 'private_fund', + 'hedge funds private equity funds non eif': 'private_fund', + 'private fund': 'private_fund', + 'private funds': 'private_fund', + 'investment club': 'business_interest', + 'ira held in cash': 'retirement_or_529', + ira: 'retirement_or_529', + 'intellectual property royalties': 'intellectual_property', + 'managed accounts e g sma and uma': 'fund', + 'managed accounts sma and uma': 'fund', + 'mutual funds': 'fund', + 'mutual fund': 'fund', + 'mineral oil solar energy rights': 'commodity_collectible', + 'ownership interest holding investments': 'business_interest', + 'ownership interest engaged in a trade or business': 'business_interest', + option: 'option', + options: 'option', + 'stock option': 'option', + 'other securities': 'other_security', + other: 'other', + pensions: 'retirement_or_529', + pension: 'retirement_or_529', + 'precious metals': 'commodity_collectible', + 'stock not publicly traded': 'private_equity', + 'non public stock': 'private_equity', + 'non-public stock': 'private_equity', + 'real estate invest trust reit': 'real_estate', + reit: 'real_estate', + 'reit eif': 'real_estate', + 'reit non eif': 'real_estate', + 'real property': 'real_estate', + 'restricted stock units rsus': 'derivative', + rsu: 'derivative', + rsus: 'derivative', + 'stock appreciation right': 'derivative', + stock: 'public_equity', + stocks: 'public_equity', + 'stocks including adrs': 'public_equity', + equity: 'public_equity', + trust: 'trust', + 'variable annuity': 'insurance_annuity', + 'variable insurance': 'insurance_annuity', + 'whole universal insurance': 'insurance_annuity', +}; + +const UNKNOWN_LABELS = new Set(['', 'unknown', 'pdf disclosed filing', 'n/a', 'na', '--', '-']); + +export function houseAssetTypeCodePattern(): string { + return Object.keys(HOUSE_ASSET_TYPE_NAMES).sort((a, b) => b.length - a.length).map(escapeRegExp).join('|'); +} + +export function assetTypeCategoryLabel(category: AssetTypeCategory): string { + return ASSET_TYPE_CATEGORY_LABELS[category] ?? ASSET_TYPE_CATEGORY_LABELS.unknown; +} + +export function isAssetTypeCategory(value: string | null | undefined): value is AssetTypeCategory { + return !!value && Object.prototype.hasOwnProperty.call(ASSET_TYPE_CATEGORY_LABELS, value); +} + +export function canonicalizeAssetType( + rawType: string | null | undefined, + rawName?: string | null, + opts?: { isOption?: boolean | null; assetName?: string | null }, +): CanonicalAssetType { + const type = cleanNullable(rawType); + const name = cleanNullable(rawName); + const upper = type?.toUpperCase() ?? null; + + if (upper && HOUSE_ASSET_TYPE_NAMES[upper]) { + const category = HOUSE_ASSET_TYPE_CATEGORIES[upper] ?? 'other'; + return { + rawType: type, + rawName: name, + code: upper, + label: HOUSE_ASSET_TYPE_NAMES[upper], + category, + categoryLabel: assetTypeCategoryLabel(category), + source: 'house_code', + }; + } + + for (const value of [name, type, opts?.assetName ?? null]) { + const key = normalizeAssetTypeKey(value); + if (!key) continue; + if (UNKNOWN_LABELS.has(key)) { + return canonical('unknown', type, name, value, 'missing'); + } + const category = LABEL_CATEGORY_ALIASES[key]; + if (category) return canonical(category, type, name, value, 'label'); + } + + if (opts?.isOption) return canonical('option', type, name, type ?? name ?? 'Option', 'option_flag'); + if (!type && !name) return canonical('unknown', null, null, null, 'missing'); + return canonical('other', type, name, type ?? name, 'unknown'); +} + +export function canonicalAssetTypeCategorySql( + rawTypeExpr: string, + rawNameExpr = 'NULL', + isOptionExpr?: string, +): string { + const raw = `lower(trim(coalesce(${rawTypeExpr}, '')))`; + const name = `lower(trim(coalesce(${rawNameExpr}, '')))`; + const upper = `upper(trim(coalesce(${rawTypeExpr}, '')))`; + const optionFlag = isOptionExpr ? `WHEN ${isOptionExpr} = 1 THEN 'option'` : ''; + const houseWhen = Object.entries(HOUSE_ASSET_TYPE_CATEGORIES) + .map(([code, category]) => `WHEN ${upper} = ${sqlQuote(code)} THEN ${sqlQuote(category)}`) + .join(' '); + const houseLabelWhen = Object.entries(HOUSE_ASSET_TYPE_CATEGORIES) + .map(([code, category]) => { + const label = HOUSE_ASSET_TYPE_NAMES[code].toLowerCase(); + return `WHEN ${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)} THEN ${sqlQuote(category)}`; + }) + .join(' '); + const labelWhen = Object.entries(LABEL_CATEGORY_ALIASES) + .map( + ([label, category]) => + `WHEN ${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)} THEN ${sqlQuote(category)}`, + ) + .join(' '); + const unknownWhen = Array.from(UNKNOWN_LABELS) + .map((label) => `${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)}`) + .join(' OR '); + return `(CASE ${optionFlag} ${houseWhen} ${houseLabelWhen} ${labelWhen} WHEN ${unknownWhen} THEN 'unknown' ELSE 'other' END)`; +} + +function canonical( + category: AssetTypeCategory, + rawType: string | null, + rawName: string | null, + label: string | null | undefined, + source: AssetTypeSource, +): CanonicalAssetType { + const categoryLabel = assetTypeCategoryLabel(category); + return { + rawType, + rawName, + code: null, + label: cleanNullable(label) ?? categoryLabel, + category, + categoryLabel, + source, + }; +} + +function normalizeAssetTypeKey(value: string | null | undefined): string { + return (value ?? '') + .toLowerCase() + .replace(/&/g, ' ') + .replace(/[^a-z0-9]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function cleanNullable(value: string | null | undefined): string | null { + const cleaned = (value ?? '').trim(); + return cleaned ? cleaned : null; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function sqlQuote(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/app/src/shared/types.ts b/app/src/shared/types.ts index 178f673dc..4c241b8af 100644 --- a/app/src/shared/types.ts +++ b/app/src/shared/types.ts @@ -6,6 +6,8 @@ * as the source of truth — downstream agents implement against these shapes. */ +import type { AssetTypeCategory } from './assetTypes'; + // --------------------------------------------------------------------------- // Primitive unions / enums // --------------------------------------------------------------------------- @@ -96,6 +98,8 @@ export interface Transaction { ticker: string | null; assetType: string | null; assetTypeName?: string | null; + assetTypeCategory?: AssetTypeCategory; + assetTypeCategoryLabel?: string; txType: TxType; amountMin: number | null; amountMax: number | null; @@ -253,7 +257,14 @@ export interface ClientTrade { companyName: string | null; /** Same-origin cached logo proxy URL for the ticker, or null when no ticker is resolved. */ logoUrl: string | null; + /** Raw disclosure asset type/code as stored in transactions.asset_type. */ type: string | null; + /** Expanded raw type name when available, e.g. House code label. */ + typeName: string | null; + /** Cross-chamber canonical instrument category, computed from raw type/code. */ + typeCategory: AssetTypeCategory; + /** Human label for typeCategory. */ + typeCategoryLabel: string; sector: string | null; marketCapBucket: string | null; }; From 5da3bb2811ff9793afd0078bc8bea1f763d86b45 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:51:26 -0500 Subject: [PATCH 02/15] feat: add chamber-aware review asset type combobox --- app/src/admin/__tests__/reviewQueue.test.ts | 3 + app/src/admin/routes.ts | 20 +- .../extraction/__tests__/normalizer.test.ts | 9 +- app/src/extraction/normalizer.ts | 2 - app/src/shared/__tests__/assetTypes.test.ts | 19 ++ app/src/shared/assetTypes.ts | 4 +- app/src/ui/__tests__/dashboardHtml.test.ts | 12 +- app/src/ui/dashboardHtml.ts | 198 ++++++++++++++---- 8 files changed, 223 insertions(+), 44 deletions(-) diff --git a/app/src/admin/__tests__/reviewQueue.test.ts b/app/src/admin/__tests__/reviewQueue.test.ts index f0f5a5a91..ab501775d 100644 --- a/app/src/admin/__tests__/reviewQueue.test.ts +++ b/app/src/admin/__tests__/reviewQueue.test.ts @@ -43,6 +43,7 @@ describe('review queue admin API', () => { 'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf', raw_object_key: 'raw/H-2026-2003695', doc_kind: 'scanned_pdf', + chamber: 'house', }, ]), } as never, @@ -55,6 +56,7 @@ describe('review queue admin API', () => { sourceUrl: string; rawObjectKey: string; docKind: string; + chamber: string; payload: { minConfidence: number; transactions: unknown[] }; }>; }; @@ -63,6 +65,7 @@ describe('review queue admin API', () => { sourceUrl: 'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf', rawObjectKey: 'raw/H-2026-2003695', docKind: 'scanned_pdf', + chamber: 'house', payload: { minConfidence: 0, transactions: [] }, }); }); diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index 8900d46f6..e6f6a5fe1 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -29,6 +29,7 @@ import { Hono } from 'hono'; import type { Env, PollConfig, PollWindow, TxType, TxSource } from '../shared/types'; import { all, get, run, type SqlParam } from '../shared/db'; +import { HOUSE_ASSET_TYPE_NAMES } from '../shared/assetTypes'; import { getConfig, setConfig } from '../shared/config'; import { uuid } from '../shared/ids'; import { listSubscriptions } from '../delivery/subscriptions'; @@ -209,6 +210,7 @@ interface ReviewRow { source_url: string | null; raw_object_key: string | null; doc_kind: string | null; + chamber?: string | null; } interface DiagnosticConnection { @@ -268,6 +270,7 @@ interface EditedTx { assetName?: string; ticker?: string | null; assetType?: string | null; + assetTypeName?: string | null; txType?: TxType; amountMin?: number | null; amountMax?: number | null; @@ -277,6 +280,15 @@ interface EditedTx { confidence?: number; } +function reviewAssetTypeName(e: EditedTx): string | null { + const supplied = typeof e.assetTypeName === 'string' ? e.assetTypeName.trim() : ''; + if (supplied) return supplied; + const raw = typeof e.assetType === 'string' ? e.assetType.trim() : ''; + if (!raw || raw.toLowerCase() === 'unknown') return null; + const code = raw.toUpperCase(); + return HOUSE_ASSET_TYPE_NAMES[code] ?? raw; +} + // --- Member photo enrichment (name -> bioguide -> unitedstates/images CDN) --- const LEGISLATOR_SOURCES = [ @@ -514,6 +526,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { f.source_url, f.raw_object_key, f.doc_kind, + f.chamber, f.ingest_status, (SELECT COUNT(*) FROM transactions t WHERE t.doc_id = rq.doc_id AND t.source = 'manual' AND t.deprecated_at IS NULL) AS manual_rows, @@ -595,6 +608,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { sourceUrl: row.source_url ?? '', rawObjectKey: row.raw_object_key ?? '', docKind: row.doc_kind ?? '', + chamber: row.chamber ?? '', models: modelsByDoc.get(row.doc_id) ?? [], }; }); @@ -726,6 +740,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const nowIso = new Date().toISOString(); for (const [rowIndex, e] of edits.entries()) { const id = uuid(); + const assetTypeName = reviewAssetTypeName(e); const rowKey = transactionRowKey(source, rowIndex, { txDate: e.txDate ?? null, owner: e.owner === 'self' || e.owner === 'spouse' || e.owner === 'joint' || e.owner === 'dependent' @@ -747,8 +762,8 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { `INSERT OR IGNORE INTO transactions ( id, doc_id, filer_id, tx_date, owner, asset_name, ticker, asset_type, tx_type, amount_min, amount_max, is_option, cap_gains_over_200, - raw_text, row_key, confidence, source, created_at, cursor_seq - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`, + raw_text, asset_type_name, row_key, confidence, source, created_at, cursor_seq + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`, [ id, docId, @@ -764,6 +779,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { e.isOption ? 1 : 0, e.capGainsOver200 ? 1 : 0, e.rawText ?? '', + assetTypeName, rowKey, e.confidence ?? 1, source, diff --git a/app/src/extraction/__tests__/normalizer.test.ts b/app/src/extraction/__tests__/normalizer.test.ts index 88fb7add5..1164c75f1 100644 --- a/app/src/extraction/__tests__/normalizer.test.ts +++ b/app/src/extraction/__tests__/normalizer.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { normalize, CONFIDENCE_THRESHOLD } from '../normalizer'; +import { normalize, CONFIDENCE_THRESHOLD, transactionRowKey } from '../normalizer'; import type { Env, Filing, ParsedTx } from '../../shared/types'; // --------------------------------------------------------------------------- @@ -101,6 +101,13 @@ const tx = (over: Partial = {}): ParsedTx => ({ }); describe('normalize', () => { + it('keeps row identity stable when asset type labels are added as enrichment', () => { + const parsed = tx({ assetType: 'Stock', assetTypeName: null }); + const enriched = tx({ assetType: 'Stock', assetTypeName: 'Stock' }); + + expect(transactionRowKey('primary', 0, enriched)).toBe(transactionRowKey('primary', 0, parsed)); + }); + it('publishes high-confidence, resolved, valid rows', async () => { const { env, cap } = makeEnv([{ ticker: 'AAPL', name: 'Apple Inc.', aliases: '["Apple"]' }]); const result = await normalize(env, filing(), [tx()]); diff --git a/app/src/extraction/normalizer.ts b/app/src/extraction/normalizer.ts index e28e89091..6dc3fad83 100644 --- a/app/src/extraction/normalizer.ts +++ b/app/src/extraction/normalizer.ts @@ -65,7 +65,6 @@ type RowKeyFields = Pick< | 'assetName' | 'ticker' | 'assetType' - | 'assetTypeName' | 'txType' | 'amountMin' | 'amountMax' @@ -95,7 +94,6 @@ export function transactionRowKey( normalizeText(fields.assetName), (fields.ticker ?? '').toUpperCase(), normalizeText(fields.assetType), - normalizeText(fields.assetTypeName ?? null), fields.txType ?? '', fields.amountMin ?? '', fields.amountMax ?? '', diff --git a/app/src/shared/__tests__/assetTypes.test.ts b/app/src/shared/__tests__/assetTypes.test.ts index 37e6054aa..3b60f9706 100644 --- a/app/src/shared/__tests__/assetTypes.test.ts +++ b/app/src/shared/__tests__/assetTypes.test.ts @@ -34,6 +34,14 @@ describe('canonicalizeAssetType', () => { expect(canonicalizeAssetType(null, null, { isOption: true }).category).toBe('option'); }); + it('lets disclosed type labels override the option flag', () => { + expect(canonicalizeAssetType('Stock', null, { isOption: true }).category).toBe('public_equity'); + }); + + it('keeps present but unmapped values separate from missing values', () => { + expect(canonicalizeAssetType('Bespoke Security').category).toBe('other'); + }); + it('treats blank and PDF placeholder rows as unknown', () => { expect(canonicalizeAssetType(null).category).toBe('unknown'); expect(canonicalizeAssetType('PDF Disclosed Filing').category).toBe('unknown'); @@ -54,4 +62,15 @@ describe('canonicalAssetTypeCategorySql', () => { expect(sql).toContain("THEN 'fixed_income_government'"); expect(sql).toContain('t.is_option = 1'); }); + + it('matches the canonical fallback order in SQL', () => { + const sql = canonicalAssetTypeCategorySql('t.asset_type', 't.asset_type_name', 't.is_option'); + expect(sql.indexOf("WHEN t.is_option = 1 THEN 'option'")).toBeGreaterThan( + sql.indexOf("THEN 'public_equity'"), + ); + expect(sql).toContain( + "WHEN lower(trim(coalesce(t.asset_type, ''))) = '' AND lower(trim(coalesce(t.asset_type_name, ''))) = '' THEN 'unknown'", + ); + expect(sql).not.toContain("lower(trim(coalesce(t.asset_type_name, ''))) = '' OR"); + }); }); diff --git a/app/src/shared/assetTypes.ts b/app/src/shared/assetTypes.ts index 088dbbb6d..48bc07131 100644 --- a/app/src/shared/assetTypes.ts +++ b/app/src/shared/assetTypes.ts @@ -326,9 +326,11 @@ export function canonicalAssetTypeCategorySql( ) .join(' '); const unknownWhen = Array.from(UNKNOWN_LABELS) + .filter(Boolean) .map((label) => `${raw} = ${sqlQuote(label)} OR ${name} = ${sqlQuote(label)}`) .join(' OR '); - return `(CASE ${optionFlag} ${houseWhen} ${houseLabelWhen} ${labelWhen} WHEN ${unknownWhen} THEN 'unknown' ELSE 'other' END)`; + const unknownCase = unknownWhen ? `WHEN ${unknownWhen} THEN 'unknown'` : ''; + return `(CASE ${houseWhen} ${houseLabelWhen} ${labelWhen} ${unknownCase} ${optionFlag} WHEN ${raw} = '' AND ${name} = '' THEN 'unknown' ELSE 'other' END)`; } function canonical( diff --git a/app/src/ui/__tests__/dashboardHtml.test.ts b/app/src/ui/__tests__/dashboardHtml.test.ts index f49b490e5..db6f87c77 100644 --- a/app/src/ui/__tests__/dashboardHtml.test.ts +++ b/app/src/ui/__tests__/dashboardHtml.test.ts @@ -210,8 +210,16 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('REVIEW_AMOUNT_BRACKETS'); expect(DASHBOARD_HTML).toContain('class="me-bracket"'); expect(DASHBOARD_HTML).toContain('class="me-asset-type"'); - expect(DASHBOARD_HTML).toContain('Stocks (ST)'); - expect(DASHBOARD_HTML).toContain('Other (OT)'); + expect(DASHBOARD_HTML).toContain('HOUSE_REVIEW_ASSET_TYPES'); + expect(DASHBOARD_HTML).toContain('SENATE_REVIEW_ASSET_TYPES'); + expect(DASHBOARD_HTML).toContain('function reviewAssetTypeDatalistId('); + expect(DASHBOARD_HTML).toContain('function reviewNormalizeAssetTypeValue('); + expect(DASHBOARD_HTML).toContain('list="'); + expect(DASHBOARD_HTML).toContain('5P'); + expect(DASHBOARD_HTML).toContain('Municipal Security'); + expect(DASHBOARD_HTML).toContain('Stock Option'); + expect(DASHBOARD_HTML).toContain('assetTypeName: reviewAssetTypeName(assetType) || null'); + expect(DASHBOARD_HTML).toContain("tr.setAttribute('data-chamber', chamber || '')"); expect(DASHBOARD_HTML).toContain('class="me-option"'); expect(DASHBOARD_HTML).toContain('Option Contract'); expect(DASHBOARD_HTML).toContain('class="me-cap"'); diff --git a/app/src/ui/dashboardHtml.ts b/app/src/ui/dashboardHtml.ts index d5633e201..5b909970d 100644 --- a/app/src/ui/dashboardHtml.ts +++ b/app/src/ui/dashboardHtml.ts @@ -438,6 +438,8 @@ export const DASHBOARD_HTML = /* html */ ` .me-row { margin: 8px 0; display:grid; grid-template-columns:minmax(80px,.6fr) 130px 165px 150px 125px 155px minmax(220px,1.8fr) minmax(210px,.9fr); gap:8px; align-items:center; } .me-row input, .me-row select { min-height:34px; width:100%; min-width:0; } .me-row .me-asset { width:100%; } + .me-asset-type-wrap { min-width:0; } + .me-asset-type-category { display:block; margin-top:2px; color:var(--text-dim); font-size:11px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .me-flags { display:flex; align-items:center; gap:6px 10px; flex-wrap:wrap; min-width:0; } .me-check { display:inline-flex; align-items:center; gap:4px; color:var(--text-dim); font-size:12px; white-space:nowrap; } .me-check input { min-height:0; width:auto; } @@ -1543,25 +1545,132 @@ function assetClassLabel(c) { if (!s) return ''; return ASSET_CLASS_UP[s.toLowerCase()] || (s.charAt(0).toUpperCase() + s.slice(1)); } -/* Friendlier label for STOCK Act asset-type codes (ST = Stocks, etc.). */ -var ASSET_TYPE_LABEL = { ST: 'Stocks', OP: 'Options', GS: 'Govt Securities', CS: 'Corporate Bonds', EF: 'Funds / ETFs', MF: 'Mutual Funds', OT: 'Other', PE: 'Private Equity', RP: 'Real Property', Unknown: 'Unclassified' }; -var REVIEW_ASSET_TYPES = [ - ['ST', 'Stocks (ST)'], - ['OP', 'Options (OP)'], - ['GS', 'Govt Securities (GS)'], - ['CS', 'Corporate Bonds (CS)'], - ['EF', 'Funds / ETFs (EF)'], - ['MF', 'Mutual Funds (MF)'], - ['OT', 'Other (OT)'], - ['PE', 'Private Equity (PE)'], - ['RP', 'Real Property (RP)'], - ['Unknown', 'Unclassified'] +/* Friendlier labels for raw STOCK Act asset-type codes / Senate eFD labels. */ +var HOUSE_REVIEW_ASSET_TYPES = [ + ['4K', '401K and Other Non-Federal Retirement Accounts', 'Retirement / 529 Accounts'], + ['5C', '529 College Savings Plan', 'Retirement / 529 Accounts'], + ['5F', '529 Portfolio', 'Retirement / 529 Accounts'], + ['5P', '529 Prepaid Tuition Plan', 'Retirement / 529 Accounts'], + ['AB', 'Asset-Backed Securities', 'Asset-Backed Securities'], + ['BA', 'Bank Accounts, Money Market Accounts and CDs', 'Cash / Bank Accounts'], + ['BK', 'Brokerage Accounts', 'Cash / Bank Accounts'], + ['CO', 'Collectibles', 'Commodities / Collectibles'], + ['CS', 'Corporate Securities (Bonds and Notes)', 'Corporate Debt'], + ['CT', 'Cryptocurrency', 'Crypto'], + ['DB', 'Defined Benefit Pension', 'Retirement / 529 Accounts'], + ['DO', 'Debts Owed to the Filer', 'Receivables'], + ['DS', 'Delaware Statutory Trust', 'Trusts'], + ['EF', 'Exchange Traded Funds (ETF)', 'Funds / ETFs / REITs'], + ['EQ', 'Excepted/Qualified Blind Trust', 'Trusts'], + ['ET', 'Exchange Traded Notes', 'Funds / ETFs / REITs'], + ['FA', 'Farms', 'Commodities / Collectibles'], + ['FE', 'Foreign Exchange Position (Currency)', 'Derivatives / Rights'], + ['FN', 'Fixed Annuity', 'Insurance / Annuities'], + ['FU', 'Futures', 'Derivatives / Rights'], + ['GS', 'Government Securities and Agency Debt', 'Government / Municipal Debt'], + ['HE', 'Hedge Funds & Private Equity Funds (EIF)', 'Private Funds'], + ['HN', 'Hedge Funds & Private Equity Funds (non-EIF)', 'Private Funds'], + ['IC', 'Investment Club', 'Business Interests'], + ['IH', 'IRA (Held in Cash)', 'Retirement / 529 Accounts'], + ['IP', 'Intellectual Property & Royalties', 'Intellectual Property'], + ['IR', 'IRA', 'Retirement / 529 Accounts'], + ['MA', 'Managed Accounts (e.g., SMA and UMA)', 'Funds / ETFs / REITs'], + ['MF', 'Mutual Funds', 'Funds / ETFs / REITs'], + ['MO', 'Mineral/Oil/Solar Energy Rights', 'Commodities / Collectibles'], + ['OI', 'Ownership Interest (Holding Investments)', 'Business Interests'], + ['OL', 'Ownership Interest (Engaged in a Trade or Business)', 'Business Interests'], + ['OP', 'Options', 'Options'], + ['OT', 'Other', 'Other'], + ['PE', 'Pensions', 'Retirement / 529 Accounts'], + ['PM', 'Precious Metals', 'Commodities / Collectibles'], + ['PS', 'Stock (Not Publicly Traded)', 'Private Equity'], + ['RE', 'Real Estate Invest. Trust (REIT)', 'Real Estate'], + ['RF', 'REIT (EIF)', 'Real Estate'], + ['RN', 'REIT (non-EIF)', 'Real Estate'], + ['RP', 'Real Property', 'Real Estate'], + ['RS', 'Restricted Stock Units (RSUs)', 'Derivatives / Rights'], + ['SA', 'Stock Appreciation Right', 'Derivatives / Rights'], + ['ST', 'Stocks (including ADRs)', 'Public Equity'], + ['TR', 'Trust', 'Trusts'], + ['VA', 'Variable Annuity', 'Insurance / Annuities'], + ['VI', 'Variable Insurance', 'Insurance / Annuities'], + ['WU', 'Whole/Universal Insurance', 'Insurance / Annuities'] ]; +var SENATE_REVIEW_ASSET_TYPES = [ + ['Stock', 'Stock', 'Public Equity'], + ['Stock Option', 'Stock Option', 'Options'], + ['Municipal Security', 'Municipal Security', 'Government / Municipal Debt'], + ['Corporate Bond', 'Corporate Bond', 'Corporate Debt'], + ['Other Securities', 'Other Securities', 'Other Securities'], + ['Non-Public Stock', 'Non-Public Stock', 'Private Equity'] +]; +var REVIEW_UNKNOWN_ASSET_TYPES = [['Unknown', 'Unclassified', 'Unknown']]; +var REVIEW_ASSET_TYPES = HOUSE_REVIEW_ASSET_TYPES.concat(SENATE_REVIEW_ASSET_TYPES).concat(REVIEW_UNKNOWN_ASSET_TYPES); +var ASSET_TYPE_LABEL = {}; +var ASSET_TYPE_CATEGORY_LABEL = {}; +REVIEW_ASSET_TYPES.forEach(function (pair) { + ASSET_TYPE_LABEL[pair[0]] = pair[1]; + ASSET_TYPE_LABEL[String(pair[0]).toUpperCase()] = pair[1]; + ASSET_TYPE_CATEGORY_LABEL[pair[0]] = pair[2]; + ASSET_TYPE_CATEGORY_LABEL[String(pair[0]).toUpperCase()] = pair[2]; +}); function assetTypeLabel(t) { var s = String(t == null ? '' : t).trim(); if (!s) return 'Unclassified'; return ASSET_TYPE_LABEL[s] || ASSET_TYPE_LABEL[s.toUpperCase()] || s; } +function reviewAssetTypeName(t) { + var s = String(t == null ? '' : t).trim(); + if (!s || s.toLowerCase() === 'unknown') return ''; + return ASSET_TYPE_LABEL[s] || ASSET_TYPE_LABEL[s.toUpperCase()] || ''; +} +function reviewNormalizeAssetTypeValue(t) { + var s = String(t == null ? '' : t).trim(); + if (!s) return ''; + var upper = s.toUpperCase(); + for (var i = 0; i < HOUSE_REVIEW_ASSET_TYPES.length; i++) { + if (HOUSE_REVIEW_ASSET_TYPES[i][0] === upper) return upper; + } + var lower = s.toLowerCase(); + for (var j = 0; j < SENATE_REVIEW_ASSET_TYPES.length; j++) { + if (String(SENATE_REVIEW_ASSET_TYPES[j][0]).toLowerCase() === lower) return SENATE_REVIEW_ASSET_TYPES[j][0]; + } + if (lower === 'unknown' || lower === 'unclassified') return 'Unknown'; + return s; +} +function reviewAssetTypeCategoryLabel(t) { + var s = String(t == null ? '' : t).trim(); + if (!s) return ''; + return ASSET_TYPE_CATEGORY_LABEL[s] || ASSET_TYPE_CATEGORY_LABEL[s.toUpperCase()] || ''; +} +function reviewAssetPairsForChamber(chamber) { + var c = String(chamber == null ? '' : chamber).toLowerCase(); + if (c === 'house') return HOUSE_REVIEW_ASSET_TYPES.concat(REVIEW_UNKNOWN_ASSET_TYPES); + if (c === 'senate') return SENATE_REVIEW_ASSET_TYPES.concat(REVIEW_UNKNOWN_ASSET_TYPES); + var seen = {}; + return REVIEW_ASSET_TYPES.filter(function (pair) { + var key = String(pair[0]).toLowerCase(); + if (seen[key]) return false; + seen[key] = true; + return true; + }); +} +function reviewAssetTypeDatalistId(chamber) { + var c = String(chamber == null ? '' : chamber).toLowerCase(); + return c === 'house' ? 'review-asset-types-house' : c === 'senate' ? 'review-asset-types-senate' : 'review-asset-types-all'; +} +function ensureReviewAssetTypeDatalists() { + if (document.getElementById('review-asset-types-all')) return; + ['house', 'senate', 'all'].forEach(function (chamber) { + var dl = document.createElement('datalist'); + dl.id = reviewAssetTypeDatalistId(chamber); + dl.innerHTML = reviewAssetPairsForChamber(chamber).map(function (pair) { + var label = pair[0] === pair[1] ? pair[2] : pair[1] + ' - ' + pair[2]; + return ''; + }).join(''); + document.body.appendChild(dl); + }); +} /* Normalize a date string to YYYY-MM-DD without timezone drift. Accepts ISO ("2026-06-15...") and US ("6/15/2026") forms (Senate filings use the latter). */ function toISODate(s) { @@ -2539,6 +2648,7 @@ function normalizeReviewEdit(t, sourceLabel) { txDate: String(t.txDate || '').slice(0, 10) || null, owner: owner, assetType: cleanAsset(t.assetType || ''), + assetTypeName: cleanAsset(t.assetTypeName || ''), isOption: Boolean(t.isOption), capGainsOver200: Boolean(t.capGainsOver200), confidence: t.confidence == null ? null : n(t.confidence), @@ -2749,16 +2859,14 @@ function amountBracketSelectHtml(tx) { }); return ''; } -function assetTypeSelectHtml(tx) { +function assetTypeInputHtml(tx, chamber) { + ensureReviewAssetTypeDatalists(); var current = String(tx.assetType || '').trim(); - var seen = {}; - var opts = ''; - REVIEW_ASSET_TYPES.forEach(function (pair) { - seen[pair[0]] = true; - opts += ''; - }); - if (current && !seen[current]) opts += ''; - return ''; + var category = reviewAssetTypeCategoryLabel(current); + return '' + + '' + + '' + esc(category) + '' + + ''; } function parseBracketValue(v) { var s = String(v || ''); @@ -2766,14 +2874,18 @@ function parseBracketValue(v) { var p = s.split(':'); return { min: p[0] === '' ? null : Number(p[0]), max: p[1] === '' ? null : Number(p[1]) }; } -function syncReviewOptionFlag(selectEl) { - var row = selectEl && selectEl.closest ? selectEl.closest('.me-row') : null; +function syncReviewAssetTypeInput(inputEl) { + var row = inputEl && inputEl.closest ? inputEl.closest('.me-row') : null; var cb = row && row.querySelector ? row.querySelector('.me-option') : null; - if (cb && selectEl.value === 'OP') cb.checked = true; + var value = inputEl ? String(inputEl.value || '').trim() : ''; + var category = reviewAssetTypeCategoryLabel(value); + if (cb && (value.toUpperCase() === 'OP' || category === 'Options')) cb.checked = true; + var label = row && row.querySelector ? row.querySelector('.me-asset-type-category') : null; + if (label) label.textContent = category || ''; } /* Shared review editor. It can start blank for manual entry, from the queued review payload, or from any selected model run. Submit stays explicit. */ -function meRowHtml(tx) { +function meRowHtml(tx, chamber) { tx = normalizeReviewEdit(tx || {}, 'review editor'); return '
' + ' ' + @@ -2781,7 +2893,7 @@ function meRowHtml(tx) { amountBracketSelectHtml(tx) + ' ' + ' ' + - assetTypeSelectHtml(tx) + + assetTypeInputHtml(tx, chamber) + '' + '' + '' + @@ -2791,21 +2903,33 @@ function meRowHtml(tx) { '' + '
'; } -function meAddRow(docId, tx) { var c = el('me-rows-' + docId); if (c) c.insertAdjacentHTML('beforeend', meRowHtml(tx)); } +function meAddRow(docId, tx) { + var c = el('me-rows-' + docId); + var tr = el('me-' + docId); + var chamber = tr ? tr.getAttribute('data-chamber') : ''; + if (c) c.insertAdjacentHTML('beforeend', meRowHtml(tx, chamber)); +} function meCancel(docId) { var tr = el('me-' + docId); if (tr) tr.parentNode.removeChild(tr); } +function reviewItemForDoc(docId) { + for (var i = 0; i < REVIEW.length; i++) { if (REVIEW[i].docId === docId) return REVIEW[i]; } + return null; +} function openQueuedReviewEditor(docId) { - var item = null; - for (var i = 0; i < REVIEW.length; i++) { if (REVIEW[i].docId === docId) { item = REVIEW[i]; break; } } + var item = reviewItemForDoc(docId); var rows = reviewPayloadTransactions(item && item.payload); - openReviewEditor(docId, rows, 'confirm', 'queued extracted rows'); + openReviewEditor(docId, rows, 'confirm', 'queued extracted rows', item && item.chamber); } function useModelRows(docId, idx) { var run = REVIEW_RUNS[docId] && REVIEW_RUNS[docId][idx]; if (!run || !run.rows || !run.rows.length) { alert('That model run has no rows to use.'); return; } - openReviewEditor(docId, run.rows, 'confirm', run.provider + ':' + run.model); + var item = reviewItemForDoc(docId); + openReviewEditor(docId, run.rows, 'confirm', run.provider + ':' + run.model, item && item.chamber); +} +function manualEntry(docId) { + var item = reviewItemForDoc(docId); + openReviewEditor(docId, [], 'manual', 'manual entry', item && item.chamber); } -function manualEntry(docId) { openReviewEditor(docId, [], 'manual', 'manual entry'); } -function openReviewEditor(docId, rows, decision, label) { +function openReviewEditor(docId, rows, decision, label, chamber) { var old = el('me-' + docId); if (old && old.parentNode) old.parentNode.removeChild(old); var row = el('rv-' + docId); @@ -2813,6 +2937,7 @@ function openReviewEditor(docId, rows, decision, label) { var tr = document.createElement('tr'); tr.id = 'me-' + docId; tr.setAttribute('data-decision', decision); + tr.setAttribute('data-chamber', chamber || ''); var safeLabel = label || (decision === 'manual' ? 'manual entry' : 'selected rows'); var title = decision === 'manual' ? 'Manual Entry' : 'Edit Rows To Confirm'; var submit = decision === 'manual' ? 'Submit Manual Entry' : 'Confirm Edited Rows'; @@ -2844,7 +2969,7 @@ function meSubmit(docId) { if (!t && !asset) return; // skip blank rows var bracket = parseBracketValue(g.querySelector('.me-bracket').value); var conf = g.querySelector('.me-conf').value; - var assetType = (g.querySelector('.me-asset-type').value || '').trim(); + var assetType = reviewNormalizeAssetTypeValue(g.querySelector('.me-asset-type').value || ''); edits.push({ ticker: t || null, assetName: asset || t || '(review entry)', @@ -2854,7 +2979,8 @@ function meSubmit(docId) { txDate: g.querySelector('.me-date').value || null, owner: g.querySelector('.me-owner').value, assetType: assetType || null, - isOption: g.querySelector('.me-option').checked || assetType === 'OP', + assetTypeName: reviewAssetTypeName(assetType) || null, + isOption: g.querySelector('.me-option').checked || assetType === 'OP' || reviewAssetTypeCategoryLabel(assetType) === 'Options', capGainsOver200: g.querySelector('.me-cap').checked, rawText: (g.querySelector('.me-raw').value || '').trim() || (decision === 'manual' ? 'manual entry' : 'review editor'), confidence: conf === '' ? (decision === 'manual' ? 1 : null) : Number(conf) From 2331de18933f38bc6eb70cdd66938dd182269882 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:10:30 -0500 Subject: [PATCH 03/15] feat: audit ingestion decisions --- app/migrations/0019_ingestion_decisions.sql | 25 ++++ app/src/admin/__tests__/reviewQueue.test.ts | 43 ++++++ app/src/admin/routes.ts | 82 ++++++++++++ app/src/extraction/agreement.ts | 19 +++ app/src/extraction/normalizer.ts | 47 ++++++- app/src/shared/ingestionDecisions.ts | 138 ++++++++++++++++++++ app/src/ui/__tests__/dashboardHtml.test.ts | 4 + app/src/ui/dashboardHtml.ts | 72 +++++++++- 8 files changed, 422 insertions(+), 8 deletions(-) create mode 100644 app/migrations/0019_ingestion_decisions.sql create mode 100644 app/src/shared/ingestionDecisions.ts diff --git a/app/migrations/0019_ingestion_decisions.sql b/app/migrations/0019_ingestion_decisions.sql new file mode 100644 index 000000000..bdeeea19a --- /dev/null +++ b/app/migrations/0019_ingestion_decisions.sql @@ -0,0 +1,25 @@ +-- 0019_ingestion_decisions.sql +-- Append-only audit trail for filing/trade publication decisions. This keeps +-- the review_queue focused on exceptions while preserving every publish/review +-- decision for admin history and future scoring/debugging. + +CREATE TABLE IF NOT EXISTS ingestion_decisions ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + action TEXT NOT NULL, + source TEXT NOT NULL, + actor TEXT, + reason TEXT, + payload TEXT, + transaction_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_doc + ON ingestion_decisions (doc_id, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_created + ON ingestion_decisions (created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_action + ON ingestion_decisions (action, created_at DESC); diff --git a/app/src/admin/__tests__/reviewQueue.test.ts b/app/src/admin/__tests__/reviewQueue.test.ts index ab501775d..dcff281db 100644 --- a/app/src/admin/__tests__/reviewQueue.test.ts +++ b/app/src/admin/__tests__/reviewQueue.test.ts @@ -100,6 +100,44 @@ describe('review queue admin API', () => { expect(body.items[0]).toMatchObject({ resolved: true, ingestStatus: 'persisted' }); }); + it('lists ingestion decision history separately from the review queue', async () => { + const res = await app.request( + '/ingestion-decisions', + { headers: { Authorization: 'Bearer admin-secret' } }, + { + ADMIN_TOKEN: 'admin-secret', + DB: fakeDb([ + { + id: 'dec-1', + doc_id: 'S-1', + action: 'auto_published', + source: 'pipeline', + actor: null, + reason: 'passed_normalization', + payload: '{"inserted":2}', + transaction_ids: '["tx1","tx2"]', + created_at: '2026-06-29T00:00:00.000Z', + chamber: 'senate', + ingest_status: 'persisted', + source_url: 'https://example/senate', + }, + ]), + } as never, + ); + expect(res.status).toBe(200); + const body = (await res.json()) as { + available: boolean; + items: Array<{ docId: string; action: string; payload: { inserted: number }; transactionIds: string[] }>; + }; + expect(body.available).toBe(true); + expect(body.items[0]).toMatchObject({ + docId: 'S-1', + action: 'auto_published', + payload: { inserted: 2 }, + transactionIds: ['tx1', 'tx2'], + }); + }); + it('unpublishes a persisted filing: soft-deletes rows, reverts, re-opens review', async () => { // fakeDb whose filing lookup resolves and whose UPDATE reports 3 retracted rows. const db = { @@ -217,12 +255,14 @@ describe('review queue admin API', () => { it("decision='manual' records hand-entered rows as source='manual'", async () => { // Capture the INSERT bind params so we can assert the source column = 'manual'. const binds: unknown[][] = []; + const auditBinds: unknown[][] = []; const db = { prepare(sql: string) { return { _sql: sql, bind(...args: unknown[]) { if (/INSERT OR IGNORE INTO transactions/.test(sql)) binds.push(args); + if (/INSERT INTO ingestion_decisions/.test(sql)) auditBinds.push(args); return this; }, async all() { @@ -259,5 +299,8 @@ describe('review queue admin API', () => { // The transactions INSERT bound source='manual' (it's the 17th positional bind). expect(binds.length).toBe(1); expect(binds[0]).toContain('manual'); + expect(auditBinds.length).toBe(1); + expect(auditBinds[0]).toContain('manual'); + expect(auditBinds[0]).toContain('admin-token'); }); }); diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index e6f6a5fe1..8a6187343 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -30,6 +30,7 @@ import { Hono } from 'hono'; import type { Env, PollConfig, PollWindow, TxType, TxSource } from '../shared/types'; import { all, get, run, type SqlParam } from '../shared/db'; import { HOUSE_ASSET_TYPE_NAMES } from '../shared/assetTypes'; +import { listIngestionDecisions, recordIngestionDecision } from '../shared/ingestionDecisions'; import { getConfig, setConfig } from '../shared/config'; import { uuid } from '../shared/ids'; import { listSubscriptions } from '../delivery/subscriptions'; @@ -112,6 +113,14 @@ function isExplicitOpenAdmin(env: EnvWithAdmin): boolean { return env.ADMIN_OPEN_IN_DEV === 'true'; } +function adminActor(c: { req: { header(name: string): string | undefined } }): string { + const accessEmail = + c.req.header('Cf-Access-Authenticated-User-Email') || + c.req.header('cf-access-authenticated-user-email'); + if (accessEmail) return accessEmail; + return c.req.header('authorization') ? 'admin-token' : 'admin'; +} + /** * Admin auth — authorized if a valid bearer token OR an allowlisted, verified * Cloudflare Access identity is presented. Open only when neither is configured. @@ -615,6 +624,25 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { return c.json({ items, count: items.length, resolved: resolved === 1 }); }); + // --- GET /ingestion-decisions ------------------------------------------ + // Append-only filing/trade decision history. Unlike review_queue, this also + // includes clean auto-published filings that never needed human review. + r.get('/ingestion-decisions', async (c) => { + const rawLimit = parseInt(c.req.query('limit') || '100', 10); + const limit = Number.isFinite(rawLimit) ? rawLimit : 100; + const docId = c.req.query('docId') || null; + try { + const items = await listIngestionDecisions(c.env.DB, { limit, docId }); + return c.json({ items, count: items.length, available: true }); + } catch (err) { + const msg = (err as Error).message; + if (/no such table|ingestion_decisions/i.test(msg)) { + return c.json({ items: [], count: 0, available: false }); + } + return c.json({ error: msg }, 500); + } + }); + // --- GET /review/:docId/extractions ------------------------------------- // Full stored readings (result_json) for one document, newest first — powers // the dashboard's "view each model's reading" panel. Separate from the list @@ -695,11 +723,24 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { } if (decision === 'reject') { + const nowIso = new Date().toISOString(); await run(c.env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [docId]); await run(c.env.DB, 'UPDATE filings SET ingest_status = ? WHERE doc_id = ?', [ 'error', docId, ]); + await recordIngestionDecision(c.env.DB, { + docId, + action: 'rejected', + source: 'admin', + actor: adminActor(c), + reason: review.reason ?? 'rejected', + payload: { + reviewCreatedAt: review.created_at, + reviewPayload: review.payload ? safeJson(review.payload) : null, + }, + createdAt: nowIso, + }); return c.json({ docId, decision: 'reject', resolved: true }); } @@ -805,6 +846,22 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { } } + await recordIngestionDecision(c.env.DB, { + docId, + action: decision === 'manual' ? 'manual' : 'confirmed', + source: 'admin', + actor: adminActor(c), + reason: review.reason ?? null, + transactionIds: insertedIds, + payload: { + source, + editCount: edits.length, + inserted: insertedIds.length, + reviewCreatedAt: review.created_at, + }, + createdAt: nowIso, + }); + return c.json({ docId, decision, @@ -862,6 +919,16 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { [docId, 'unpublished: ' + reason, null, nowIso], ); + await recordIngestionDecision(c.env.DB, { + docId, + action: 'unpublished', + source: 'admin', + actor: adminActor(c), + reason, + payload: { deprecatedTransactions: deprecated }, + createdAt: nowIso, + }); + return c.json({ docId, unpublished: true, deprecatedTransactions: deprecated, reason }); }); @@ -1987,6 +2054,21 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { // 0018_agreement_attempted.sql — one autonomous agreement attempt per review doc. 'ALTER TABLE review_queue ADD COLUMN agreement_attempted_at TEXT', 'CREATE INDEX IF NOT EXISTS idx_review_queue_agreement ON review_queue (agreement_attempted_at)', + // 0019_ingestion_decisions.sql — append-only audit trail for publication/review decisions. + `CREATE TABLE IF NOT EXISTS ingestion_decisions ( + id TEXT PRIMARY KEY, + doc_id TEXT NOT NULL, + action TEXT NOT NULL, + source TEXT NOT NULL, + actor TEXT, + reason TEXT, + payload TEXT, + transaction_ids TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL + )`, + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_doc ON ingestion_decisions (doc_id, created_at DESC)', + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_created ON ingestion_decisions (created_at DESC)', + 'CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_action ON ingestion_decisions (action, created_at DESC)', ]; const applied: string[] = []; const skipped: string[] = []; diff --git a/app/src/extraction/agreement.ts b/app/src/extraction/agreement.ts index 5df4397d8..613bbebab 100644 --- a/app/src/extraction/agreement.ts +++ b/app/src/extraction/agreement.ts @@ -16,6 +16,7 @@ import { runCandidateOnDoc, type BakeoffCandidate, type CandidateDocResult } fro import { arbitrationRowKey } from '../extractors/types'; import { recomputeTransactions, persistTransactions, HARD_FAILURE_FLAGS } from './normalizer'; import { mapFiling, type FilingRow } from '../delivery/rows'; +import { recordIngestionDecision } from '../shared/ingestionDecisions'; export interface AgreementModels { a: BakeoffCandidate; @@ -102,6 +103,24 @@ export async function processAgreementDoc( const insertedIds = await persistTransactions(env, txs); await run(env.DB, "UPDATE filings SET ingest_status = 'persisted', error = NULL WHERE doc_id = ?", [docId]); await run(env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [docId]); + if (insertedIds.length > 0) { + await recordIngestionDecision(env.DB, { + docId, + action: 'agreement_published', + source: 'agreement', + reason: 'model_agreement', + transactionIds: insertedIds, + payload: { + rowCount: flagged.length, + inserted: insertedIds.length, + models: { + a: `${models.a.provider}:${models.a.model}`, + b: `${models.b.provider}:${models.b.model}`, + ...(models.c ? { c: `${models.c.provider}:${models.c.model}` } : {}), + }, + }, + }); + } for (const txId of insertedIds) { try { await env.DELIVERY_QUEUE.send({ type: 'delivery.dispatch', txId }); } catch { /* best-effort */ } } diff --git a/app/src/extraction/normalizer.ts b/app/src/extraction/normalizer.ts index 6dc3fad83..20404d8b4 100644 --- a/app/src/extraction/normalizer.ts +++ b/app/src/extraction/normalizer.ts @@ -23,6 +23,7 @@ import { all, run, fromBool, parseJson } from '../shared/db'; import { isValidBracket, matchBracket, nearestBracket } from '../shared/brackets'; import { canonicalizeAssetType } from '../shared/assetTypes'; import { uuid } from '../shared/ids'; +import { recordIngestionDecision } from '../shared/ingestionDecisions'; import { isPlaceholderTicker, resolveTickerDeterministic, TICKER_ALIASES } from './tickerNormalize'; /** @@ -184,10 +185,24 @@ export async function normalize( ); if (needsReview) { + const reason = reviewReason(flagged, minConfidence); await routeToReview(env, filing, flagged, minConfidence, nowIso, { extractor: extractorName, modelVersion, }); + await recordIngestionDecision(env.DB, { + docId: filing.docId, + action: 'review_opened', + source: 'pipeline', + reason, + payload: { + minConfidence, + extractor: extractorName, + modelVersion, + transactionCount: transactions.length, + }, + createdAt: nowIso, + }); return { transactions, minConfidence, needsReview: true }; } @@ -197,6 +212,23 @@ export async function normalize( await run(env.DB, 'UPDATE review_queue SET resolved = 1 WHERE doc_id = ?', [filing.docId]); const insertedIds = await persistTransactions(env, transactions); + if (insertedIds.length > 0) { + await recordIngestionDecision(env.DB, { + docId: filing.docId, + action: 'auto_published', + source: 'pipeline', + reason: 'passed_normalization', + transactionIds: insertedIds, + payload: { + minConfidence, + extractor: extractorName, + modelVersion, + transactionCount: transactions.length, + inserted: insertedIds.length, + }, + createdAt: nowIso, + }); + } // Fan out delivery only for rows D1 actually inserted. Retries/concurrent // normalizations hit the unique row key and are ignored without duplicate @@ -504,12 +536,7 @@ async function routeToReview( nowIso: string, meta: { extractor: string | null; modelVersion: string | null }, ): Promise { - const reasons = new Set(); - for (const f of flagged) for (const flag of f.flags) reasons.add(flag); - if (flagged.length === 0) reasons.add('no_transactions_extracted'); - if (minConfidence < CONFIDENCE_THRESHOLD) reasons.add('low_confidence'); - - const reason = Array.from(reasons).join(',') || 'needs_review'; + const reason = reviewReason(flagged, minConfidence); const payload = JSON.stringify({ minConfidence, extractor: meta.extractor, @@ -536,6 +563,14 @@ async function routeToReview( ); } +function reviewReason(flagged: FlaggedTx[], minConfidence: number): string { + const reasons = new Set(); + for (const f of flagged) for (const flag of f.flags) reasons.add(flag); + if (flagged.length === 0) reasons.add('no_transactions_extracted'); + if (minConfidence < CONFIDENCE_THRESHOLD) reasons.add('low_confidence'); + return Array.from(reasons).join(',') || 'needs_review'; +} + // --------------------------------------------------------------------------- // Small helpers // --------------------------------------------------------------------------- diff --git a/app/src/shared/ingestionDecisions.ts b/app/src/shared/ingestionDecisions.ts new file mode 100644 index 000000000..28b7c089c --- /dev/null +++ b/app/src/shared/ingestionDecisions.ts @@ -0,0 +1,138 @@ +import { all, parseJson, run, type SqlParam } from './db'; +import { uuid } from './ids'; + +export type IngestionDecisionAction = + | 'auto_published' + | 'review_opened' + | 'confirmed' + | 'manual' + | 'rejected' + | 'unpublished' + | 'agreement_published'; + +export type IngestionDecisionSource = 'pipeline' | 'admin' | 'agreement'; + +export interface IngestionDecisionInput { + docId: string; + action: IngestionDecisionAction; + source: IngestionDecisionSource; + actor?: string | null; + reason?: string | null; + payload?: Record | null; + transactionIds?: string[]; + createdAt?: string; +} + +interface IngestionDecisionRow { + id: string; + doc_id: string; + action: string; + source: string; + actor: string | null; + reason: string | null; + payload: string | null; + transaction_ids: string | null; + created_at: string; + chamber?: string | null; + ingest_status?: string | null; + source_url?: string | null; +} + +export interface ListedIngestionDecision { + id: string; + docId: string; + action: string; + source: string; + actor: string | null; + reason: string | null; + payload: Record | null; + transactionIds: string[]; + createdAt: string; + chamber: string | null; + ingestStatus: string | null; + sourceUrl: string | null; +} + +/** + * Best-effort audit write. The decision trail should never block ingestion or + * admin remediation if a deployment reaches code before the D1 migration. + */ +export async function recordIngestionDecision( + db: D1Database, + input: IngestionDecisionInput, +): Promise { + const id = uuid(); + const createdAt = input.createdAt ?? new Date().toISOString(); + try { + await run( + db, + `INSERT INTO ingestion_decisions + (id, doc_id, action, source, actor, reason, payload, transaction_ids, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + id, + input.docId, + input.action, + input.source, + input.actor ?? null, + input.reason ?? null, + input.payload ? JSON.stringify(input.payload) : null, + JSON.stringify(input.transactionIds ?? []), + createdAt, + ], + ); + return id; + } catch (err) { + console.warn( + 'ingestion decision audit write skipped:', + input.docId, + input.action, + (err as Error).message, + ); + return null; + } +} + +export async function listIngestionDecisions( + db: D1Database, + opts: { limit?: number; docId?: string | null } = {}, +): Promise { + const requestedLimit = Number.isFinite(opts.limit) ? Math.floor(opts.limit as number) : 100; + const limit = Math.min(Math.max(requestedLimit, 1), 200); + const where: string[] = []; + const params: SqlParam[] = []; + if (opts.docId) { + where.push('d.doc_id = ?'); + params.push(opts.docId); + } + params.push(limit); + + const rows = await all( + db, + `SELECT + d.id, d.doc_id, d.action, d.source, d.actor, d.reason, d.payload, + d.transaction_ids, d.created_at, + f.chamber, f.ingest_status, f.source_url + FROM ingestion_decisions d + LEFT JOIN filings f ON f.doc_id = d.doc_id + ${where.length ? `WHERE ${where.join(' AND ')}` : ''} + ORDER BY d.created_at DESC + LIMIT ?`, + params, + ); + + return rows.map((row) => ({ + id: row.id, + docId: row.doc_id, + action: row.action, + source: row.source, + actor: row.actor ?? null, + reason: row.reason ?? null, + payload: parseJson | null>(row.payload, null), + transactionIds: parseJson(row.transaction_ids, []), + createdAt: row.created_at, + chamber: row.chamber ?? null, + ingestStatus: row.ingest_status ?? null, + sourceUrl: row.source_url ?? null, + })); +} diff --git a/app/src/ui/__tests__/dashboardHtml.test.ts b/app/src/ui/__tests__/dashboardHtml.test.ts index db6f87c77..bce0d9661 100644 --- a/app/src/ui/__tests__/dashboardHtml.test.ts +++ b/app/src/ui/__tests__/dashboardHtml.test.ts @@ -203,6 +203,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('function useModelRows('); expect(DASHBOARD_HTML).toContain('function openReviewEditor('); expect(DASHBOARD_HTML).toContain('Review / Confirm'); + expect(DASHBOARD_HTML).toContain('Decision History'); + expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=100'"); + expect(DASHBOARD_HTML).toContain('function renderDecisionHistory('); + expect(DASHBOARD_HTML).toContain('var DECISIONS'); expect(DASHBOARD_HTML).toContain('Use This Model'); expect(DASHBOARD_HTML).toContain('Bake-Off Runs ('); expect(DASHBOARD_HTML).toContain('Queued Extracted Rows'); diff --git a/app/src/ui/dashboardHtml.ts b/app/src/ui/dashboardHtml.ts index 5b909970d..4a0484cff 100644 --- a/app/src/ui/dashboardHtml.ts +++ b/app/src/ui/dashboardHtml.ts @@ -1286,6 +1286,13 @@ export const DASHBOARD_HTML = /* html */ `

Confirm promotes the read to the live feed; Manual lets you hand-key the rows (recorded as source=manual) when the automated read is wrong or too low-confidence; Reject discards it. Models / readings come from extraction_runs (populated by POST /api/admin/bakeoff). POST /api/admin/review/:docId {decision}

+
+

Decision History

+ + + +
TimeDocActionSourceReasonRows
+
@@ -1472,6 +1479,7 @@ export const DASHBOARD_HTML = /* html */ ` /* ============================ STATE ============================ */ var TRADES = []; // live transactions (newest first) var REVIEW = []; // review-queue items +var DECISIONS = []; // ingestion decision audit rows var REVIEW_RUNS = {}; // docId -> full extraction runs loaded on demand var SCHEDULE = []; // PollWindow[] var aggressive = false; @@ -2565,11 +2573,70 @@ function loadReview() { // API HOOK: GET /api/admin/review-queue?resolved= return fetch('/api/admin/review-queue?resolved=' + REVIEW_RESOLVED, { headers: adminHeaders() }) .then(okOrThrow) - .then(function (data) { REVIEW = data.items || []; renderReview(); }) + .then(function (data) { REVIEW = data.items || []; renderReview(); loadDecisionHistory(); }) .catch(function (e) { el('reviewBody').innerHTML = stateRow(6, isAuthError(e) ? ADMIN_MOVED_MSG : ('Could not load review queue: ' + e.message)); }); } +function loadDecisionHistory() { + // API HOOK: GET /api/admin/ingestion-decisions + return fetch('/api/admin/ingestion-decisions?limit=100', { headers: adminHeaders() }) + .then(okOrThrow) + .then(function (data) { + DECISIONS = data.items || []; + renderDecisionHistory(data.available !== false); + }) + .catch(function (e) { + var body = el('decisionBody'); + if (body) body.innerHTML = stateRow(6, isAuthError(e) ? ADMIN_MOVED_MSG : ('Could not load decision history: ' + e.message)); + }); +} +function decisionActionLabel(action) { + return String(action || '').replace(/_/g, ' '); +} +function decisionRowsText(d) { + var ids = d && Array.isArray(d.transactionIds) ? d.transactionIds : []; + if (!ids.length) return '—'; + return ids.length + ' row' + (ids.length === 1 ? '' : 's'); +} +function decisionReasonText(d) { + var reason = d && d.reason ? String(d.reason) : ''; + var payload = d && d.payload && typeof d.payload === 'object' ? d.payload : null; + var bits = []; + if (reason) bits.push(reason.replace(/_/g, ' ')); + if (payload && typeof payload.minConfidence === 'number') bits.push('conf ' + Math.round(payload.minConfidence * 100) + '%'); + if (payload && typeof payload.inserted === 'number') bits.push('inserted ' + payload.inserted); + if (payload && typeof payload.deprecatedTransactions === 'number') bits.push('retracted ' + payload.deprecatedTransactions); + return bits.join(' · ') || '—'; +} +function decisionDocHtml(d) { + var docId = d.docId || ''; + var url = safeDocUrl(d.sourceUrl); + if (!url) return '' + esc(docId) + ''; + return '' + esc(docId) + ''; +} +function renderDecisionHistory(available) { + var body = el('decisionBody'); + if (!body) return; + if (!available) { + body.innerHTML = stateRow(6, 'Decision history is not migrated yet.'); + return; + } + if (!DECISIONS.length) { + body.innerHTML = stateRow(6, 'No decisions recorded yet.'); + return; + } + body.innerHTML = DECISIONS.map(function (d) { + return '' + + '' + esc(dateTimeText(d.createdAt)) + '' + + '' + decisionDocHtml(d) + '' + + '' + statusBadge(decisionActionLabel(d.action)) + '' + + '' + esc([d.source || '', d.actor || ''].filter(Boolean).join(' · ')) + '' + + '' + esc(decisionReasonText(d)) + '' + + '' + esc(decisionRowsText(d)) + '' + + ''; + }).join(''); +} /* Translate review reason codes + payload into plain English for non-engineers. */ var REASON_LABELS = { low_confidence: 'Automated read below publish threshold', @@ -2823,6 +2890,7 @@ function resolveReview(docId, decision) { .then(function () { if (isUnpublish) { loadReview(); } // item returns to pending; reload current tab else { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); renderReview(); } + loadDecisionHistory(); loadFeed(); }) .catch(function (e) { @@ -2993,7 +3061,7 @@ function meSubmit(docId) { body: JSON.stringify({ decision: decision, edits: edits }) }) .then(okOrThrow) - .then(function () { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); if (tr && tr.parentNode) tr.parentNode.removeChild(tr); renderReview(); loadFeed(); }) + .then(function () { REVIEW = REVIEW.filter(function (x) { return x.docId !== docId; }); if (tr && tr.parentNode) tr.parentNode.removeChild(tr); renderReview(); loadDecisionHistory(); loadFeed(); }) .catch(function (e) { if (tr) tr.querySelectorAll('button,input,select').forEach(function (b) { b.disabled = false; }); alert(isAuthError(e) ? ADMIN_MOVED_MSG : ('Review submit failed: ' + e.message)); From 0ce705a19782b70b2135488b597c32c62df4be47 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:14:58 -0500 Subject: [PATCH 04/15] fix: preserve asset type compatibility edges --- app/src/admin/routes.ts | 1 + app/src/export/__tests__/pitScores.test.ts | 27 +++++++++++++++++++ app/src/export/pitScores.ts | 8 +++++- .../extraction/__tests__/normalizer.test.ts | 4 +++ app/src/extraction/normalizer.ts | 8 ++++++ app/src/shared/__tests__/assetTypes.test.ts | 3 +++ app/src/shared/assetTypes.ts | 16 ++++++++--- 7 files changed, 62 insertions(+), 5 deletions(-) diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index 8a6187343..d86a36c06 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -790,6 +790,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { assetName: e.assetName ?? '', ticker: e.ticker ?? null, assetType: e.assetType ?? null, + assetTypeName, txType: (e.txType as TxType) ?? 'P', amountMin: e.amountMin ?? null, amountMax: e.amountMax ?? null, diff --git a/app/src/export/__tests__/pitScores.test.ts b/app/src/export/__tests__/pitScores.test.ts index fe24e4846..79d1f2e14 100644 --- a/app/src/export/__tests__/pitScores.test.ts +++ b/app/src/export/__tests__/pitScores.test.ts @@ -179,6 +179,33 @@ describe('buildPitScoreExport pagination', () => { expect(second.pagination.nextCursor).toBeNull(); }); + it('uses securities_ref asset class when disclosure asset type is missing', async () => { + const env = { + DB: fakeDb({ + transactions: [ + tx('tx1', 'AAPL', '2026-01-01T00:00:00.000Z', { + asset_type: null, + asset_type_name: null, + asset_class: 'equity', + }), + ], + price_eod: [], + spx_eod: [], + }), + }; + const result = await buildPitScoreExport( + env as never, + { limit: 1, format: 'json', placebo: 'none', source: 'all' }, + new Date('2026-03-01T00:00:00.000Z'), + ); + + expect(result.rows[0]).toMatchObject({ + assetType: 'equity', + assetTypeCategory: 'public_equity', + assetTypeCategorySource: 'label', + }); + }); + it('flags seed/date-only rows as not true historical validation rows', async () => { const env = { DB: fakeDb({ diff --git a/app/src/export/pitScores.ts b/app/src/export/pitScores.ts index ace88469f..39fdaeb0a 100644 --- a/app/src/export/pitScores.ts +++ b/app/src/export/pitScores.ts @@ -1202,10 +1202,16 @@ async function buildRow( const signedScore = score == null ? null : direction === 'SELL' ? -score : direction === 'BUY' ? score : 0; const price = await (priceCache.get(ticker) ?? priceCache.set(ticker, priceSeries(env, ticker)).get(ticker)!); const ref = txs.find((t) => t.company_name || t.cik || t.asset_class || t.sector) ?? txs[0]; - const canonicalAssetType = canonicalizeAssetType(ref?.asset_type ?? null, ref?.asset_type_name ?? null, { + const disclosureAssetType = canonicalizeAssetType(ref?.asset_type ?? null, ref?.asset_type_name ?? null, { isOption: txs.some((t) => t.is_option === 1), assetName: ref?.asset_name ?? null, }); + const canonicalAssetType = disclosureAssetType.category === 'unknown' && ref?.asset_class + ? canonicalizeAssetType(ref.asset_class, ref?.asset_type_name ?? null, { + isOption: txs.some((t) => t.is_option === 1), + assetName: ref?.asset_name ?? null, + }) + : disclosureAssetType; const includedDisclosures = txs.map((t) => { const disclosureAssetType = canonicalizeAssetType(t.asset_type, t.asset_type_name, { isOption: t.is_option === 1, diff --git a/app/src/extraction/__tests__/normalizer.test.ts b/app/src/extraction/__tests__/normalizer.test.ts index 1164c75f1..2279e51b8 100644 --- a/app/src/extraction/__tests__/normalizer.test.ts +++ b/app/src/extraction/__tests__/normalizer.test.ts @@ -106,6 +106,10 @@ describe('normalize', () => { const enriched = tx({ assetType: 'Stock', assetTypeName: 'Stock' }); expect(transactionRowKey('primary', 0, enriched)).toBe(transactionRowKey('primary', 0, parsed)); + + const house = tx({ assetType: 'ST', assetTypeName: 'Stocks (including ADRs)' }); + const houseWithoutLabel = tx({ assetType: 'ST', assetTypeName: null }); + expect(transactionRowKey('primary', 0, house)).not.toBe(transactionRowKey('primary', 0, houseWithoutLabel)); }); it('publishes high-confidence, resolved, valid rows', async () => { diff --git a/app/src/extraction/normalizer.ts b/app/src/extraction/normalizer.ts index 20404d8b4..31cb5c752 100644 --- a/app/src/extraction/normalizer.ts +++ b/app/src/extraction/normalizer.ts @@ -66,6 +66,7 @@ type RowKeyFields = Pick< | 'assetName' | 'ticker' | 'assetType' + | 'assetTypeName' | 'txType' | 'amountMin' | 'amountMax' @@ -95,6 +96,7 @@ export function transactionRowKey( normalizeText(fields.assetName), (fields.ticker ?? '').toUpperCase(), normalizeText(fields.assetType), + rowKeyAssetTypeName(fields.assetType, fields.assetTypeName ?? null), fields.txType ?? '', fields.amountMin ?? '', fields.amountMax ?? '', @@ -110,6 +112,12 @@ export function transactionRowKey( return `v1:${source}:${rowIndex}:${fnv1a32(payload)}`; } +function rowKeyAssetTypeName(assetType: string | null | undefined, assetTypeName: string | null): string { + const type = normalizeText(assetType ?? null); + const name = normalizeText(assetTypeName); + return name && name !== type ? name : ''; +} + /** * Re-derive Transaction rows (with the current, recalibrated confidence rubric) * from parsed rows WITHOUT any DB write or delivery fan-out. Shared by normalize() diff --git a/app/src/shared/__tests__/assetTypes.test.ts b/app/src/shared/__tests__/assetTypes.test.ts index 3b60f9706..2626521c8 100644 --- a/app/src/shared/__tests__/assetTypes.test.ts +++ b/app/src/shared/__tests__/assetTypes.test.ts @@ -45,6 +45,9 @@ describe('canonicalizeAssetType', () => { it('treats blank and PDF placeholder rows as unknown', () => { expect(canonicalizeAssetType(null).category).toBe('unknown'); expect(canonicalizeAssetType('PDF Disclosed Filing').category).toBe('unknown'); + expect(canonicalizeAssetType('N/A').category).toBe('unknown'); + expect(canonicalizeAssetType('-').category).toBe('unknown'); + expect(canonicalizeAssetType('--').category).toBe('unknown'); expect(assetTypeCategoryLabel('unknown')).toBe('Unknown'); }); diff --git a/app/src/shared/assetTypes.ts b/app/src/shared/assetTypes.ts index 48bc07131..22ddcdc1f 100644 --- a/app/src/shared/assetTypes.ts +++ b/app/src/shared/assetTypes.ts @@ -250,7 +250,7 @@ const LABEL_CATEGORY_ALIASES: Record = { 'whole universal insurance': 'insurance_annuity', }; -const UNKNOWN_LABELS = new Set(['', 'unknown', 'pdf disclosed filing', 'n/a', 'na', '--', '-']); +const UNKNOWN_LABELS = new Set(['', 'unknown', 'pdf disclosed filing', 'n/a', 'n a', 'na', '--', '-']); export function houseAssetTypeCodePattern(): string { return Object.keys(HOUSE_ASSET_TYPE_NAMES).sort((a, b) => b.length - a.length).map(escapeRegExp).join('|'); @@ -287,11 +287,11 @@ export function canonicalizeAssetType( } for (const value of [name, type, opts?.assetName ?? null]) { - const key = normalizeAssetTypeKey(value); - if (!key) continue; - if (UNKNOWN_LABELS.has(key)) { + if (isUnknownAssetTypeValue(value)) { return canonical('unknown', type, name, value, 'missing'); } + const key = normalizeAssetTypeKey(value); + if (!key) continue; const category = LABEL_CATEGORY_ALIASES[key]; if (category) return canonical(category, type, name, value, 'label'); } @@ -361,6 +361,14 @@ function normalizeAssetTypeKey(value: string | null | undefined): string { .trim(); } +function isUnknownAssetTypeValue(value: string | null | undefined): boolean { + if (value == null) return false; + const raw = value.trim().toLowerCase(); + if (UNKNOWN_LABELS.has(raw)) return true; + const key = normalizeAssetTypeKey(value); + return key ? UNKNOWN_LABELS.has(key) : false; +} + function cleanNullable(value: string | null | undefined): string | null { const cleaned = (value ?? '').trim(); return cleaned ? cleaned : null; From 0591091d87216501cbd3e467ef7eadc80ed321a0 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:17:40 -0500 Subject: [PATCH 05/15] Add admin market data diagnostics --- app/src/admin/__tests__/diagnostics.test.ts | 70 ++++++++++ app/src/admin/routes.ts | 144 +++++++++++++++++++- 2 files changed, 213 insertions(+), 1 deletion(-) diff --git a/app/src/admin/__tests__/diagnostics.test.ts b/app/src/admin/__tests__/diagnostics.test.ts index a6f1aa468..d0f59136b 100644 --- a/app/src/admin/__tests__/diagnostics.test.ts +++ b/app/src/admin/__tests__/diagnostics.test.ts @@ -26,6 +26,33 @@ function fakeDb() { ] as T[], }; } + if (/FROM securities_ref/i.test(sql) && /CASE\s+WHEN lower\(source\)/i.test(sql)) { + return { + results: [ + { + provider: 'massive', + calls_total: 2, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24T11:30:00.000Z', + errors_last_24h: 0, + }, + ] as T[], + }; + } + if (/FROM securities_ref/i.test(sql) && /COUNT\(\*\) AS calls_total/i.test(sql)) { + return { + results: [ + { + calls_total: 5, + calls_last_24h: 2, + calls_today: 1, + last_used_at: '2026-06-24T11:00:00.000Z', + errors_last_24h: 1, + }, + ] as T[], + }; + } if (/FROM securities_ref/i.test(sql) && /enrichment_error/i.test(sql)) { return { results: [ @@ -37,6 +64,42 @@ function fakeDb() { ] as T[], }; } + if (/FROM price_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 7, + calls_last_24h: 20, + calls_today: 3, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM spx_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 100, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM tx_performance/i.test(sql)) { + return { + results: [ + { + calls_total: 50, + calls_last_24h: 10, + calls_today: 5, + last_used_at: '2026-06-24T12:30:00.000Z', + }, + ] as T[], + }; + } if (/FROM deliveries/i.test(sql)) return { results: [] as T[] }; if (/FROM review_queue/i.test(sql)) return { results: [] as T[] }; if (/FROM client_commands/i.test(sql)) return { results: [] as T[] }; @@ -72,6 +135,8 @@ describe('admin diagnostics API', () => { ADMIN_TOKEN: 'admin-secret', GEMINI_API_KEY: 'gemini-secret', FMP_API_KEY: 'fmp-secret', + MASSIVE_API_KEY: 'massive-secret', + PRICE_PROVIDER: 'massive', DB: fakeDb(), } as never, ); @@ -91,10 +156,15 @@ describe('admin diagnostics API', () => { callsToday: 1, }), expect.objectContaining({ id: 'source:house', status: 'ok', callsToday: 2 }), + expect.objectContaining({ id: 'provider:massive', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:prices', status: 'ok', configured: true, callsToday: 3 }), + expect.objectContaining({ id: 'cache:spx', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:performance', status: 'ok', configured: true, callsToday: 5 }), ]), ); expect(JSON.stringify(body)).not.toContain('gemini-secret'); expect(JSON.stringify(body)).not.toContain('fmp-secret'); + expect(JSON.stringify(body)).not.toContain('massive-secret'); expect(body.errors).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index d86a36c06..e30713597 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -981,6 +981,12 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const env = c.env as Env & { GEMINI_API_KEY?: string; FMP_API_KEY?: string; + MASSIVE_API_KEY?: string; + INTRINIO_API_KEY?: string; + TWELVEDATA_API_KEY?: string; + FINNHUB_API_KEY?: string; + LOGODEV_PUBLISHABLE_KEY?: string; + PRICE_PROVIDER?: string; WEBHOOK_SIGNING_KEY?: string; GOOGLE_OAUTH_CLIENT_ID?: string; RESEND_API_KEY?: string; @@ -1074,7 +1080,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { const fmpRow = fmp[0]; connections.push({ id: 'provider:fmp', - label: 'FMP Market Data', + label: 'FMP Enrichment', status: connectionStatus(!!env.FMP_API_KEY, fmpRow?.errors_last_24h ?? 0, fmpRow?.last_used_at ?? null), configured: !!env.FMP_API_KEY, lastUsedAt: fmpRow?.last_used_at ?? null, @@ -1085,6 +1091,142 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { note: env.FMP_API_KEY ? 'Enrichment rows refreshed' : 'FMP_API_KEY is not configured', }); + const providerRows = await optionalAll<{ + provider: string; + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + errors_last_24h: number; + }>( + c.env, + `SELECT CASE + WHEN lower(source) LIKE '%massive%' THEN 'massive' + WHEN lower(source) LIKE '%intrinio%' THEN 'intrinio' + WHEN lower(source) LIKE '%twelvedata%' THEN 'twelvedata' + WHEN lower(source) LIKE '%finnhub%' THEN 'finnhub' + WHEN lower(source) LIKE '%edgar%' THEN 'edgar' + ELSE 'other' + END AS provider, + COUNT(*) AS calls_total, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(enriched_at) AS last_used_at, + SUM(CASE WHEN enrichment_error IS NOT NULL AND enrichment_error != '' AND enriched_at >= ? THEN 1 ELSE 0 END) AS errors_last_24h + FROM securities_ref + WHERE source IS NOT NULL AND source != '' + GROUP BY provider`, + [last24, today, last24], + ); + const providerUsage = new Map(providerRows.map((row) => [row.provider, row])); + const addMarketProvider = (id: string, label: string, configured: boolean, note: string) => { + const row = providerUsage.get(id); + connections.push({ + id: `provider:${id}`, + label, + status: connectionStatus(configured, row?.errors_last_24h ?? 0, row?.last_used_at ?? null), + configured, + lastUsedAt: row?.last_used_at ?? null, + callsTotal: row?.calls_total ?? 0, + callsLast24h: row?.calls_last_24h ?? 0, + callsToday: row?.calls_today ?? 0, + errorsLast24h: row?.errors_last_24h ?? 0, + note, + }); + }; + addMarketProvider('massive', 'Massive Market Data', !!env.MASSIVE_API_KEY, env.MASSIVE_API_KEY ? 'Reference/price fallback configured' : 'MASSIVE_API_KEY is not configured'); + addMarketProvider('intrinio', 'Intrinio Reference Data', !!env.INTRINIO_API_KEY, env.INTRINIO_API_KEY ? 'Reference fallback configured' : 'INTRINIO_API_KEY is not configured'); + addMarketProvider('twelvedata', 'Twelve Data Reference', !!env.TWELVEDATA_API_KEY, env.TWELVEDATA_API_KEY ? 'Reference fallback configured' : 'TWELVEDATA_API_KEY is not configured'); + addMarketProvider('finnhub', 'Finnhub Reference', !!env.FINNHUB_API_KEY, env.FINNHUB_API_KEY ? 'Reference fallback configured' : 'FINNHUB_API_KEY is not configured'); + addMarketProvider('edgar', 'SEC EDGAR Reference', true, 'Free fallback; no secret required'); + + const priceRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(DISTINCT ticker) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM price_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const priceRow = priceRows[0]; + const hasPriceProvider = !!(env.FMP_API_KEY || env.MASSIVE_API_KEY); + connections.push({ + id: 'cache:prices', + label: 'Asset Price Cache', + status: connectionStatus(hasPriceProvider, 0, priceRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: priceRow?.last_used_at ?? null, + callsTotal: priceRow?.calls_total ?? 0, + callsLast24h: priceRow?.calls_last_24h ?? 0, + callsToday: priceRow?.calls_today ?? 0, + errorsLast24h: 0, + note: hasPriceProvider + ? `PRICE_PROVIDER=${env.PRICE_PROVIDER || 'fmp'}; counts show cached assets/rows, not raw API calls` + : 'No FMP_API_KEY or MASSIVE_API_KEY configured for price history', + }); + + const spxRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM spx_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const spxRow = spxRows[0]; + connections.push({ + id: 'cache:spx', + label: 'S&P Benchmark Cache', + status: connectionStatus(hasPriceProvider, 0, spxRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: spxRow?.last_used_at ?? null, + callsTotal: spxRow?.calls_total ?? 0, + callsLast24h: spxRow?.calls_last_24h ?? 0, + callsToday: spxRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'SPY-adjusted close history used as the S&P comparison baseline', + }); + + const perfRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(computed_at) AS last_used_at + FROM tx_performance`, + [last24, today], + ); + const perfRow = perfRows[0]; + connections.push({ + id: 'cache:performance', + label: 'Trade Performance Anchors', + status: connectionStatus(hasPriceProvider, 0, perfRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: perfRow?.last_used_at ?? null, + callsTotal: perfRow?.calls_total ?? 0, + callsLast24h: perfRow?.calls_last_24h ?? 0, + callsToday: perfRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'Required for per-trade and member S&P-relative performance', + }); + const webhooks = await optionalAll<{ calls_total: number; calls_last_24h: number; From ce7014b1eb306906b937d60bb4241154f22845ff Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:33:31 -0500 Subject: [PATCH 06/15] Add cross-app capabilities manifest --- app/docs/fmp-data-sharing.md | 15 ++- app/src/export/__tests__/routes.test.ts | 33 ++++++ app/src/export/routes.ts | 151 +++++++++++++++++++++++- 3 files changed, 196 insertions(+), 3 deletions(-) diff --git a/app/docs/fmp-data-sharing.md b/app/docs/fmp-data-sharing.md index 91a45be80..82ddfbe2a 100644 --- a/app/docs/fmp-data-sharing.md +++ b/app/docs/fmp-data-sharing.md @@ -318,6 +318,20 @@ provider — see the cross-app data-sharing plan. ## Bulk snapshot export (full-history bootstrap / catch-up) +Before App B hardcodes a new route or export shape, it should read the +machine-readable integration manifest: + +``` +GET https://congress.trade/api/export/capabilities +Headers: Authorization: Bearer +``` + +The response lists the current cross-app contract version, supported import +payload slots, import limits, read endpoints, PIT score export settings, +bulk-snapshot table names, placebo exports, and whether the App B return path is +configured. It intentionally reports only boolean secret/config status; it never +echoes token values or peer URLs. + The per-ticker reads above are for incremental, one-symbol cache-aside. To **bootstrap from scratch** or **catch up after a downtime gap**, App B can pull a daily, date-partitioned NDJSON snapshot of the whole market-data set instead of @@ -339,7 +353,6 @@ schema, and a per-table `downloadPath`: { "generatedAt": "2026-06-25T04:01:00.000Z", "snapshotDate": "2026-06-25", - "snapshotDate": "2026-06-25", "runId": "9f3c…", // unique per run; pinned into downloadPath "format": "ndjson", "tables": { diff --git a/app/src/export/__tests__/routes.test.ts b/app/src/export/__tests__/routes.test.ts index 688b47be9..5f2f304b6 100644 --- a/app/src/export/__tests__/routes.test.ts +++ b/app/src/export/__tests__/routes.test.ts @@ -115,6 +115,39 @@ describe('GET /api/export/bulk-snapshot — auth', () => { }); }); +describe('GET /api/export/capabilities', () => { + it('401 without the scoped ingest token', async () => { + expect((await req('/capabilities', baseEnv())).status).toBe(401); + }); + + it('returns the cross-app contract without leaking configured secret values', async () => { + const res = await req( + '/capabilities', + baseEnv({ + APP_B_IMPORT_URL: 'https://app-b.example/api/admin/securities/import', + APP_B_INGEST_TOKEN: 'peer-secret', + IMPORT_MAX_REFS: '123', + }), + TOKEN, + ); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).not.toContain(TOKEN); + expect(text).not.toContain('peer-secret'); + expect(text).not.toContain('https://app-b.example'); + const body = JSON.parse(text) as { + contractVersion: string; + configured: { ingestToken: boolean; appBReturnPath: boolean }; + endpoints: { imports: { securities: { limits: { refs: number }; accepts: string[] } }; exports: { pitScores: { scoreVersion: string } } }; + }; + expect(body.contractVersion).toBe('congress-trade-crossapp-v1'); + expect(body.configured).toEqual({ ingestToken: true, appBReturnPath: true }); + expect(body.endpoints.imports.securities.limits.refs).toBe(123); + expect(body.endpoints.imports.securities.accepts).toContain('fundamentals'); + expect(body.endpoints.exports.pitScores.scoreVersion).toBe('congress-pit-v2'); + }); +}); + describe('GET /api/export/congress-pit-scores', () => { it('401 without a token', async () => { expect((await req('/congress-pit-scores', baseEnv())).status).toBe(401); diff --git a/app/src/export/routes.ts b/app/src/export/routes.ts index e256b8599..7c19b923d 100644 --- a/app/src/export/routes.ts +++ b/app/src/export/routes.ts @@ -40,14 +40,37 @@ import { buildPitScoreExport, parsePitScoreQuery, pitScoreRowsToNdjson, + PIT_PLACEBOS, + PIT_SCORE_VERSION, } from './pitScores'; -/** Env augmented with the scoped cross-app ingest token (mirrors admin/routes). */ -type ExportEnv = Env & { INGEST_TOKEN?: string }; +/** Env augmented with cross-app sharing config (mirrors admin/share routes). */ +type ExportEnv = Env & { INGEST_TOKEN?: string; APP_B_IMPORT_URL?: string; APP_B_INGEST_TOKEN?: string }; const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; const RUN_ID_RE = /^[A-Za-z0-9-]{1,64}$/; // crypto.randomUUID() shape (hex + dashes) const VALID_TABLES = new Set(SNAPSHOT_TABLES.map((t) => t.name)); +const CAPABILITIES_VERSION = 'congress-trade-crossapp-v1'; + +const IMPORT_DEFAULT_LIMITS = { + bytes: 1_500_000, + refs: 2_000, + spx: 5_000, + prices: 100, + closesPerTicker: 1_500, + insider: 5_000, + shortVolume: 5_000, +}; + +const IMPORT_MAX_LIMITS = { + bytes: 3_000_000, + refs: 5_000, + spx: 10_000, + prices: 250, + closesPerTicker: 3_000, + insider: 10_000, + shortVolume: 10_000, +}; function todayUtc(now = new Date()): string { return now.toISOString().slice(0, 10); @@ -93,9 +116,133 @@ function shapeManifest(manifest: SnapshotManifest, tables: SnapshotTableName[]): }; } +function positiveIntSetting(raw: string | undefined, fallback: number, max: number): number { + const n = Number.parseInt(raw ?? '', 10); + if (!Number.isFinite(n) || n <= 0) return fallback; + return Math.min(Math.floor(n), max); +} + +function integrationImportLimits(env: ExportEnv): typeof IMPORT_DEFAULT_LIMITS { + return { + bytes: positiveIntSetting(env.IMPORT_MAX_BYTES, IMPORT_DEFAULT_LIMITS.bytes, IMPORT_MAX_LIMITS.bytes), + refs: positiveIntSetting(env.IMPORT_MAX_REFS, IMPORT_DEFAULT_LIMITS.refs, IMPORT_MAX_LIMITS.refs), + spx: positiveIntSetting(env.IMPORT_MAX_SPX, IMPORT_DEFAULT_LIMITS.spx, IMPORT_MAX_LIMITS.spx), + prices: positiveIntSetting(env.IMPORT_MAX_PRICES, IMPORT_DEFAULT_LIMITS.prices, IMPORT_MAX_LIMITS.prices), + closesPerTicker: positiveIntSetting( + env.IMPORT_MAX_CLOSES_PER_TICKER, + IMPORT_DEFAULT_LIMITS.closesPerTicker, + IMPORT_MAX_LIMITS.closesPerTicker, + ), + insider: positiveIntSetting(env.IMPORT_MAX_INSIDER, IMPORT_DEFAULT_LIMITS.insider, IMPORT_MAX_LIMITS.insider), + shortVolume: positiveIntSetting(env.IMPORT_MAX_SHORT_VOLUME, IMPORT_DEFAULT_LIMITS.shortVolume, IMPORT_MAX_LIMITS.shortVolume), + }; +} + +function integrationCapabilities(env: ExportEnv): Record { + const configured = { + ingestToken: Boolean(env.INGEST_TOKEN), + appBReturnPath: Boolean(env.APP_B_IMPORT_URL && env.APP_B_INGEST_TOKEN), + }; + return { + app: 'congress.trade', + generatedAt: new Date().toISOString(), + contractVersion: CAPABILITIES_VERSION, + auth: { + scheme: 'bearer', + tokenName: 'INGEST_TOKEN', + requiredFor: [ + '/api/admin/securities/import', + '/api/export/capabilities', + '/api/export/congress-pit-scores', + '/api/export/bulk-snapshot', + '/api/export/bulk-snapshot/file', + ], + }, + configured, + peerSharing: { + role: 'source-of-truth-for-congressional-disclosures-and-point-in-time-scores', + appBReturnPathConfigured: configured.appBReturnPath, + appBImportUrlConfigured: Boolean(env.APP_B_IMPORT_URL), + appBIngestTokenConfigured: Boolean(env.APP_B_INGEST_TOKEN), + noEchoPolicy: 'Only freshly fetched local deltas are pushed to App B; App B-origin imports are not echoed back.', + }, + endpoints: { + imports: { + securities: { + method: 'POST', + path: '/api/admin/securities/import', + auth: 'bearer INGEST_TOKEN', + accepts: ['refs', 'prices', 'spx', 'insider', 'shortVolume', 'fundamentals', 'analyst', 'origin'], + limits: integrationImportLimits(env), + }, + }, + publicReads: { + marketBundle: { method: 'GET', path: '/api/market/bundle/:ticker?from=&to=' }, + marketRef: { method: 'GET', path: '/api/market/ref/:ticker' }, + marketRefs: { method: 'GET', path: '/api/market/refs?tickers=AAPL,MSFT' }, + prices: { method: 'GET', path: '/api/market/prices/:ticker?from=&to=' }, + spx: { method: 'GET', path: '/api/market/spx?from=&to=' }, + insider: { method: 'GET', path: '/api/market/insider/:ticker?from=&to=' }, + shortVolume: { method: 'GET', path: '/api/market/short-volume/:ticker?from=&to=' }, + fundamentals: { method: 'GET', path: '/api/market/fundamentals/:ticker?from=&to=' }, + analyst: { method: 'GET', path: '/api/market/analyst/:ticker?from=&to=' }, + transactions: { method: 'GET', path: '/api/transactions?cursor=&limit=&member=&ticker=&type=&chamber=' }, + }, + analytics: { + tickerLeaderboard: { method: 'GET', path: '/api/analytics/ticker-leaderboard?window=&rankBy=' }, + clusterBuys: { method: 'GET', path: '/api/analytics/cluster-buys?window=' }, + memberLeaderboard: { method: 'GET', path: '/api/analytics/member-leaderboard?window=&rankBy=' }, + memberPerformance: { method: 'GET', path: '/api/analytics/member/:filerId/performance?from=&to=' }, + conviction: { method: 'GET', path: '/api/analytics/conviction?ticker=&window=' }, + tickerBacktest: { method: 'GET', path: '/api/analytics/ticker/:ticker/backtest?from=&to=' }, + conflicts: { method: 'GET', path: '/api/analytics/conflicts?ticker=§or=' }, + }, + exports: { + pitScores: { + method: 'GET', + path: '/api/export/congress-pit-scores?from=&to=&ticker=&cursor=&limit=&format=json|ndjson&placebo=&source=&minConf=', + auth: 'bearer INGEST_TOKEN', + scoreVersion: PIT_SCORE_VERSION, + maxLimit: 500, + placebosAvailable: PIT_PLACEBOS, + }, + bulkSnapshot: { + method: 'GET', + path: '/api/export/bulk-snapshot?date=&tables=&format=ndjson', + auth: 'bearer INGEST_TOKEN', + format: 'ndjson', + tables: SNAPSHOT_TABLES.map((t) => ({ name: t.name, keyColumns: t.keyCols })), + }, + bulkSnapshotFile: { + method: 'GET', + path: '/api/export/bulk-snapshot/file?date=&runId=&table=', + auth: 'bearer INGEST_TOKEN', + format: 'ndjson', + }, + }, + }, + recommendedSync: { + bootstrap: 'Pull /api/export/bulk-snapshot, persist manifest runId/objectKeys, then stream each downloadPath.', + incrementalMarketData: 'Use /api/market/* reads as a cache-aside tier before paid providers.', + congressionalSignals: 'Use /api/export/congress-pit-scores for historical validation and /api/analytics/* for live overlays.', + writeBack: 'POST newly fetched refs/prices/spx/enrichment deltas to /api/admin/securities/import with origin set by the sender.', + }, + }; +} + export function buildExportRouter(): Hono<{ Bindings: ExportEnv }> { const r = new Hono<{ Bindings: ExportEnv }>(); + // --- GET /capabilities -------------------------------------------------- + // Token-gated machine-readable integration contract for sibling apps. App B + // can use this before hardcoding a new route, limit, or export shape. + r.get('/capabilities', async (c) => { + if (!(await isAuthorized(c.env, c.req.header('authorization')))) { + return c.json({ error: 'unauthorized' }, 401); + } + return c.json(integrationCapabilities(c.env)); + }); + // --- GET /congress-pit-scores ------------------------------------------ // Token-gated point-in-time score export for App B historical validation. // Emits one row per (ticker, disclosure availability timestamp) observation. From e99944bad79bc3cdec9e0415b901d8874b341303 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 14:51:39 -0500 Subject: [PATCH 07/15] Wire @jaywedgeworth22/congress-trading-shared shared package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace duplicated types (Chamber, Owner, TxType, SecurityRef, MktCapBucket) with imports from the cross-app shared package. Add contracts.ts barrel re-exporting all shared types, schemas, constants, and utilities. Files changed: - src/shared/types.ts — import Chamber/Owner/TxType from shared - src/enrichment/types.ts — import SecurityRef/MktCapBucket from shared - src/enrichment/compute.ts — import marketCapBucket from shared - src/analytics/compute.ts — import bracketMidpoint from shared - src/analytics/sql.ts — import PartyBucket/WINDOW_PRESETS from shared - src/shared/contracts.ts (new) — barrel re-exports for shared package - package.json — add shared package dep Verification: npx tsc --noEmit — 0 errors Docs: docs/rollouts/2026-06-29-congress-trading-shared.md Co-authored-by: Cursor --- app/package-lock.json | 10 +++++- app/package.json | 1 + app/src/analytics/compute.ts | 15 ++++---- app/src/analytics/sql.ts | 11 ++++-- app/src/enrichment/compute.ts | 13 ++----- app/src/enrichment/types.ts | 32 ++---------------- app/src/shared/contracts.ts | 64 +++++++++++++++++++++++++++++++++++ app/src/shared/types.ts | 14 ++------ 8 files changed, 97 insertions(+), 63 deletions(-) create mode 100644 app/src/shared/contracts.ts diff --git a/app/package-lock.json b/app/package-lock.json index fbad12165..74a54af55 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,6 +8,7 @@ "name": "congress-feed", "version": "0.1.0", "dependencies": { + "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", @@ -1173,6 +1174,14 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@jaywedgeworth22/congress-trading-shared": { + "version": "1.0.0", + "resolved": "git+ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git#c34cfae038ee32d8bb3711fc0365f4f6d4ea585b", + "license": "UNLICENSED", + "dependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -3088,7 +3097,6 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/app/package.json b/app/package.json index 4d4da8b80..821a3beae 100644 --- a/app/package.json +++ b/app/package.json @@ -17,6 +17,7 @@ "preview:deploy": "bash scripts/deploy-preview.sh" }, "dependencies": { + "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", diff --git a/app/src/analytics/compute.ts b/app/src/analytics/compute.ts index 1dd2b74e4..8823a6b53 100644 --- a/app/src/analytics/compute.ts +++ b/app/src/analytics/compute.ts @@ -10,17 +10,16 @@ */ import { computePerformance } from '../prices/compute'; +import { bracketMidpoint } from "@jaywedgeworth22/congress-trading-shared"; + +// Re-export for backward compatibility. +export { bracketMidpoint }; /** - * Estimated dollar value of one STOCK Act bracket. Mirror of - * BRACKET_MIDPOINT_SQL: midpoint of [min,max]; open top tier (max == null) → - * floor (min); missing amount → 0. + * Estimated dollar value of one STOCK Act bracket. Delegate to the shared + * cross-app implementation for consistency with Agentic Trading. */ -export function bracketMidpoint(min: number | null, max: number | null): number { - if (max != null && min != null) return (min + max) / 2; - if (min != null) return min; // open-ended top tier ($50M+) or max missing - return 0; -} +// bracketMidpoint is now imported from the shared package (above). /** * Buy/sell sentiment in [0,1]: share of directional (P+S) activity that is diff --git a/app/src/analytics/sql.ts b/app/src/analytics/sql.ts index a4d76508a..a39ae365c 100644 --- a/app/src/analytics/sql.ts +++ b/app/src/analytics/sql.ts @@ -22,6 +22,10 @@ import type { Chamber, TxType } from '../shared/types'; import type { SqlParam } from '../shared/db'; +import type { PartyBucket } from "@jaywedgeworth22/congress-trading-shared"; +import { WINDOW_PRESETS } from "@jaywedgeworth22/congress-trading-shared"; + +export { type PartyBucket, WINDOW_PRESETS }; // --------------------------------------------------------------------------- // Enumerations + validators (closed sets → safe to interpolate as literals) @@ -32,7 +36,9 @@ import type { SqlParam } from '../shared/db'; * the presets below, but any positive `d` is valid — so callers can request a * custom age (e.g. ?window=45d) without enumerating it here. */ -export const WINDOW_PRESETS = ['1d', '7d', '30d', '90d', '180d', '365d', '1825d', 'all'] as const; +// WINDOW_PRESETS is now imported from the shared package. +// Local Window type + validators are kept here (not shared). + export type Window = string; // always produced via asWindow(): 'all' | `${number}d` const WINDOW_RE = /^(\d{1,5})d$/; const MAX_WINDOW_DAYS = 36500; // ~100y guardrail against absurd inputs @@ -64,7 +70,8 @@ export function asSourceFilter(v: unknown, fallback: SourceFilter = 'all'): Sour : fallback; } -export type PartyBucket = 'D' | 'R' | 'O'; +// PartyBucket is now imported from the shared package and re-exported above. + export function asPartyBucket(v: unknown): PartyBucket | undefined { if (typeof v !== 'string' || v.length === 0) return undefined; const c = v[0].toUpperCase(); diff --git a/app/src/enrichment/compute.ts b/app/src/enrichment/compute.ts index 7342855df..e2de20973 100644 --- a/app/src/enrichment/compute.ts +++ b/app/src/enrichment/compute.ts @@ -10,17 +10,10 @@ */ import type { MktCapBucket, SecurityRef } from './types'; +import { marketCapBucket } from "@jaywedgeworth22/congress-trading-shared"; -/** Bucket a USD market cap into the standard size tiers. null for missing/≤0. */ -export function marketCapBucket(n: number | null | undefined): MktCapBucket | null { - if (n == null || !Number.isFinite(n) || n <= 0) return null; - if (n >= 200e9) return 'mega'; - if (n >= 10e9) return 'large'; - if (n >= 2e9) return 'mid'; - if (n >= 300e6) return 'small'; - if (n >= 50e6) return 'micro'; - return 'nano'; -} +// Re-export for backward compatibility. +export { marketCapBucket }; /** * Map a numeric SIC code to a coarse sector via the SEC's SIC division ranges. diff --git a/app/src/enrichment/types.ts b/app/src/enrichment/types.ts index f7aff852e..e0a8edf6a 100644 --- a/app/src/enrichment/types.ts +++ b/app/src/enrichment/types.ts @@ -8,36 +8,8 @@ * fail soft (return null) so a missing key / unknown ticker never throws. */ -/** Market-cap size bucket (standard industry thresholds). */ -export type MktCapBucket = 'mega' | 'large' | 'mid' | 'small' | 'micro' | 'nano'; - -/** Reference data for one security, keyed by ticker (mirrors securities_ref). */ -export interface SecurityRef { - ticker: string; - companyName: string | null; - sector: string | null; - industry: string | null; - /** equity | etf | adr | fund | other */ - assetClass: string | null; - isEtf: boolean; - isAdr: boolean; - country: string | null; - stateHq: string | null; - stateOfIncorp: string | null; - exchange: string | null; - exchangeShort: string | null; - currency: string | null; - marketCap: number | null; - marketCapBucket: MktCapBucket | null; - /** Shares outstanding, so market cap can be recomputed from the latest close. */ - sharesOutstanding: number | null; - ipoDate: string | null; - cik: string | null; - sicCode: string | null; - sicDescription: string | null; - /** Which provider(s) produced this row. */ - source: string | null; -} +import type { MktCapBucket, SecurityRef } from "@jaywedgeworth22/congress-trading-shared"; +export type { MktCapBucket, SecurityRef }; /** * A single enrichment source. `fetchRef` returns the fields it can resolve for a diff --git a/app/src/shared/contracts.ts b/app/src/shared/contracts.ts new file mode 100644 index 000000000..7357adeea --- /dev/null +++ b/app/src/shared/contracts.ts @@ -0,0 +1,64 @@ +// Re-exports from @jaywedgeworth22/congress-trading-shared with local aliases +// for backward compatibility. Prefer importing from the shared package directly +// in new code. + +export { + // Types + type CongressTransaction, + type TransactionsPage, + type TransactionsQuery, + type SecurityRef, + type PriceClose, + type PriceSeries, + type BundleResponse, + type FundamentalRow, + type AnalystRow, + type InsiderRow, + type ShortVolumeRow, + type TickerLeader, + type ClusterBuy, + type MemberLeader, + type MemberPerformance, + type ConvictionTicker, + type BacktestHorizon, + type TickerBacktest, + type CommitteeConflict, + type SharePayload, + type CongressEvent, + type CongressEventType, + type Chamber, + type PartyBucket, + type Owner, + type TxType, + type MktCapBucket, + type SnapshotManifest, + type SnapshotTableInfo, + // Schemas + CongressTransactionSchema, + TransactionsPageSchema, + SecurityRefSchema, + PriceCloseSchema, + SharePayloadSchema, + CongressEventSchema, + ConvictionTickerSchema, + parseArray, + parseSafe, + // Constants + TICKER_ALIASES, + MKT_CAP_THRESHOLDS, + API_PATHS, + WINDOW_PRESETS, + LAG_BUCKETS, + DEFAULT_CONGRESS_TRADE_BASE_URL, + DEFAULT_TRANSACTIONS_LIMIT, + MAX_REFS_BATCH, + APP_B_ORIGIN_TAG, + // Utils + normalizeTicker, + resolveTickerAlias, + marketCapBucket, + bracketMidpoint, + isIsoDate, + daysBetween, + mergeRefs, +} from "@jaywedgeworth22/congress-trading-shared"; diff --git a/app/src/shared/types.ts b/app/src/shared/types.ts index b2e893f48..d2b704760 100644 --- a/app/src/shared/types.ts +++ b/app/src/shared/types.ts @@ -7,12 +7,8 @@ */ import type { AssetTypeCategory } from './assetTypes'; - -// --------------------------------------------------------------------------- -// Primitive unions / enums -// --------------------------------------------------------------------------- - -export type Chamber = 'house' | 'senate'; +import type { Chamber, Owner, TxType } from "@jaywedgeworth22/congress-trading-shared"; +export type { Chamber, Owner, TxType }; /** * Filing type code. STOCK Act Periodic Transaction Reports are 'P'. @@ -34,12 +30,6 @@ export type IngestStatus = /** Detected physical form of a disclosure document. */ export type DocKind = 'senate_html' | 'text_pdf' | 'scanned_pdf' | 'unknown'; -/** Beneficial owner of a transaction. */ -export type Owner = 'self' | 'spouse' | 'joint' | 'dependent'; - -/** Transaction type: Purchase | Sale | Exchange. */ -export type TxType = 'P' | 'S' | 'E'; - /** Delivery transport for a subscription. */ export type DeliveryChannel = 'webhook' | 'sse'; From 49c34d68daba63769ca6397d3fbc8a7b4cc37797 Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:07:13 -0500 Subject: [PATCH 08/15] fix: make shared package installable in ci --- .github/workflows/ci.yml | 12 ++++++++++++ app/package-lock.json | 5 +++-- app/package.json | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6932585d5..6bb2a4684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,18 @@ jobs: cache: npm cache-dependency-path: app/package-lock.json + - name: Configure private GitHub dependencies + env: + GH_PAT: ${{ secrets.GH_PAT }} + run: | + if [ -z "$GH_PAT" ]; then + echo "::error::GH_PAT with read access to jaywedgeworth22/congress-trading-shared is required for npm ci." + exit 1 + fi + git config --global url."https://x-access-token:${GH_PAT}@github.com/".insteadOf "https://github.com/" + git config --global url."https://x-access-token:${GH_PAT}@github.com/".insteadOf "ssh://git@github.com/" + git config --global url."https://x-access-token:${GH_PAT}@github.com/".insteadOf "git@github.com:" + - name: Install dependencies run: npm ci diff --git a/app/package-lock.json b/app/package-lock.json index 74a54af55..7bd103ffb 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,7 +8,7 @@ "name": "congress-feed", "version": "0.1.0", "dependencies": { - "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", + "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#9924dd614d643dc655bbee9788d15f32f48760dc", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", @@ -1176,7 +1176,8 @@ }, "node_modules/@jaywedgeworth22/congress-trading-shared": { "version": "1.0.0", - "resolved": "git+ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git#c34cfae038ee32d8bb3711fc0365f4f6d4ea585b", + "resolved": "git+ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git#9924dd614d643dc655bbee9788d15f32f48760dc", + "integrity": "sha512-RpU03bLum4N6W14ootXr5lgNYCmMRw8wSd98zBx5FkMleYZIeROxKxCbUZPfbAJ6OknRtTrAgi5RtgbMbJOeSg==", "license": "UNLICENSED", "dependencies": { "zod": "^3.23.8" diff --git a/app/package.json b/app/package.json index 821a3beae..f322306f2 100644 --- a/app/package.json +++ b/app/package.json @@ -17,7 +17,7 @@ "preview:deploy": "bash scripts/deploy-preview.sh" }, "dependencies": { - "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", + "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#9924dd614d643dc655bbee9788d15f32f48760dc", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", From 2e613cb2df10b6d7b8c030b9803e229bb8d9e4ff Mon Sep 17 00:00:00 2001 From: Jay Wedgeworth <12656028+jaywedgeworth22@users.noreply.github.com> Date: Mon, 29 Jun 2026 20:24:15 -0500 Subject: [PATCH 09/15] chore: harden app update workflows --- .github/dependabot.yml | 18 ++ .github/workflows/ci.yml | 5 +- .github/workflows/codeql.yml | 28 +++ .github/workflows/codex-autofix.yml | 100 ++++---- .github/workflows/deploy-staging.yml | 75 ++++++ .github/workflows/deploy.yml | 49 ++++ .github/workflows/security.yml | 51 +++++ .github/workflows/uptime-monitor.yml | 43 ++++ app/.dev.vars.example | 4 + app/package-lock.json | 52 ++++- app/package.json | 2 +- app/src/analytics/compute.ts | 15 +- app/src/analytics/sql.ts | 11 +- app/src/delivery/__tests__/queueRetry.test.ts | 8 + app/src/enrichment/compute.ts | 13 +- app/src/enrichment/types.ts | 32 ++- app/src/index.ts | 84 ++++--- app/src/shared/contracts.ts | 64 ------ app/src/shared/types.ts | 16 +- app/src/ui/__tests__/dashboardHtml.test.ts | 27 ++- app/src/ui/dashboardHtml.ts | 215 +++++++++++++----- app/wrangler.toml | 5 + 22 files changed, 670 insertions(+), 247 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/deploy-staging.yml create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/uptime-monitor.yml delete mode 100644 app/src/shared/contracts.ts diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4381c8769 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: "/app" + schedule: + interval: daily + time: "08:00" + open-pull-requests-limit: 10 + groups: + cloudflare: + patterns: + - "@cloudflare/*" + - "wrangler" + - "hono" + testing: + patterns: + - "vitest" + - "@playwright/*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6932585d5..81c1f0815 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: app/package-lock.json @@ -34,3 +34,6 @@ jobs: - name: Test run: npm test + + - name: Audit + run: npm audit diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..39ff9dcaa --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,28 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '0 6 * * 1' # Monday at 6 AM UTC + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + contents: read + strategy: + fail-fast: false + matrix: + language: [javascript-typescript] + steps: + - uses: actions/checkout@v4 + - uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + - uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/codex-autofix.yml b/.github/workflows/codex-autofix.yml index f4db50ddd..389a69931 100644 --- a/.github/workflows/codex-autofix.yml +++ b/.github/workflows/codex-autofix.yml @@ -1,6 +1,8 @@ name: Codex Autofix # Autonomous responder to the Codex PR reviewer (chatgpt-codex-connector[bot]). +# Calls the shared reusable workflow in congress-trading-shared; the prompt +# (repo-specific behaviour) stays here so it's auditable in-repo. # # Roles are DISTINCT so the two bots never compete to "review first": # • Codex = reviewer — fires on every push, posts P1/P2 suggestions. @@ -8,7 +10,7 @@ name: Codex Autofix # then pushes. The push makes Codex review again → # clean ping-pong, capped to avoid an infinite loop. # -# Prerequisites (one-time, see the PR description): +# Prerequisites: # 1. Secret ANTHROPIC_API_KEY (Settings → Secrets and variables → Actions). # 2. A token whose pushes RE-TRIGGER CI + Codex. The default GITHUB_TOKEN does # NOT re-trigger workflows, so EITHER install the Claude GitHub App @@ -40,65 +42,51 @@ concurrency: jobs: autofix: - # Only when the Codex bot posted the feedback (review / inline comment / PR - # comment), or a maintainer dispatched it manually. issue_comment must be on - # a PR (issue.pull_request != null), never a plain issue. if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_review' && github.event.review.user.login == 'chatgpt-codex-connector[bot]') || (github.event_name == 'pull_request_review_comment' && github.event.comment.user.login == 'chatgpt-codex-connector[bot]') || (github.event_name == 'issue_comment' && github.event.issue.pull_request != null && github.event.comment.user.login == 'chatgpt-codex-connector[bot]') - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }} + uses: jaywedgeworth22/congress-trading-shared/.github/workflows/codex-autofix-reusable.yml@main + with: + allowed_bots: "chatgpt-codex-connector,chatgpt-codex-connector[bot]" + prompt: | + You are the autonomous fixer that responds to the Codex PR reviewer + (chatgpt-codex-connector[bot]) on THIS pull request. You do NOT review + the PR yourself — Codex is the reviewer; you only address its feedback. - - uses: anthropics/claude-code-action@v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }} - # The whole policy lives in the prompt so behavior is auditable in-repo. - prompt: | - You are the autonomous fixer that responds to the Codex PR reviewer - (chatgpt-codex-connector[bot]) on THIS pull request. You do NOT review - the PR yourself — Codex is the reviewer; you only address its feedback. + Repo: Congress.Trade (Cloudflare Worker; the app lives in `app/`). + Verify before committing: `cd app && npm run typecheck && npm test`. + Read app/AGENTS.md and app/CLAUDE.md first — follow them as the source + of truth (migrations, deploy gates, do-not-deploy rules). The git + author email MUST be 12656028+jaywedgeworth22@users.noreply.github.com. + Do NOT deploy, run remote D1 migrations, or run production crawlers. - Repo: Congress.Trade (Cloudflare Worker; the app lives in `app/`). - Verify before committing: `cd app && npm run typecheck && npm test`. - Read app/AGENTS.md and app/CLAUDE.md first — follow them as the source - of truth (migrations, deploy gates, do-not-deploy rules). The git - author email MUST be 12656028+jaywedgeworth22@users.noreply.github.com. - Do NOT deploy, run remote D1 migrations, or run production crawlers. - - Do this: - 1. ROUND CAP: count commits on the PR branch whose message contains - "[codex-autofix]". If there are already 10 or more, STOP: post one PR - comment summarizing the remaining open Codex items and asking the - maintainer how to proceed, then end without further changes. - 2. Read the PR's review threads. Separate OUTDATED threads (anchored to - code already changed — usually already fixed) from genuinely NEW, - non-outdated Codex items. - 3. For each NEW item: if it is a clear correctness bug OR a simple - cosmetic/doc fix, fix it. If it is ambiguous or architecturally - significant, do NOT guess — post a PR comment asking the maintainer, - and skip it. - 4. If `git merge origin/main` is needed (branch behind main), merge it - and resolve conflicts. If a migration is added, add SQL under - app/migrations/ and update POST /api/admin/migrate per app/AGENTS.md. - 5. Run `cd app && npm run typecheck && npm test`. Only commit if it - passes. Commit message must start with "[codex-autofix] ". Push to - the PR branch. - 6. When the PR is functional and you have addressed the actionable - items, ensure auto-merge is enabled: - `gh pr merge --squash --auto`. Do NOT use --admin and do NOT - try to bypass any required check. - 7. Be frugal with PR comments — only comment to ask the maintainer a - question, to report the round cap was hit, or to flag a finding you - are intentionally not fixing. The diff is the record otherwise. - claude_args: | - --max-turns 60 - --allowedTools "Edit,Write,Read,Bash" - allowed_bots: "chatgpt-codex-connector,chatgpt-codex-connector[bot]" + Do this: + 1. ROUND CAP: count commits on the PR branch whose message contains + "[codex-autofix]". If there are already 10 or more, STOP: post one PR + comment summarizing the remaining open Codex items and asking the + maintainer how to proceed, then end without further changes. + 2. Read the PR's review threads. Separate OUTDATED threads (anchored to + code already changed — usually already fixed) from genuinely NEW, + non-outdated Codex items. + 3. For each NEW item: if it is a clear correctness bug OR a simple + cosmetic/doc fix, fix it. If it is ambiguous or architecturally + significant, do NOT guess — post a PR comment asking the maintainer, + and skip it. + 4. If `git merge origin/main` is needed (branch behind main), merge it + and resolve conflicts. If a migration is added, add SQL under + app/migrations/ and update POST /api/admin/migrate per app/AGENTS.md. + 5. Run `cd app && npm run typecheck && npm test`. Only commit if it + passes. Commit message must start with "[codex-autofix] ". Push to + the PR branch. + 6. When the PR is functional and you have addressed the actionable + items, ensure auto-merge is enabled: + `gh pr merge --squash --auto`. Do NOT use --admin and do NOT + try to bypass any required check. + 7. Be frugal with PR comments — only comment to ask the maintainer a + question, to report the round cap was hit, or to flag a finding you + are intentionally not fixing. The diff is the record otherwise. + secrets: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GH_PAT: ${{ secrets.GH_PAT }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 000000000..6fbbeb103 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,75 @@ +name: Deploy Preview + +on: + push: + branches: [staging] + workflow_dispatch: {} + +# Never run two preview deploys at once; let an in-flight deploy finish. +concurrency: + group: deploy-preview + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build isolated preview config + run: | + cp wrangler.preview.example.toml wrangler.preview.toml + python - <<'PY' + from pathlib import Path + import os + + path = Path("wrangler.preview.toml") + text = path.read_text() + replacements = { + "PREVIEW_D1_DATABASE_ID": os.environ["PREVIEW_D1_DATABASE_ID"], + "PREVIEW_KV_NAMESPACE_ID": os.environ["PREVIEW_KV_NAMESPACE_ID"], + "https://congress-trade-preview..workers.dev": os.environ["PREVIEW_APP_BASE_URL"], + } + for old, new in replacements.items(): + text = text.replace(old, new) + path.write_text(text) + PY + env: + PREVIEW_D1_DATABASE_ID: ${{ secrets.PREVIEW_D1_DATABASE_ID }} + PREVIEW_KV_NAMESPACE_ID: ${{ secrets.PREVIEW_KV_NAMESPACE_ID }} + PREVIEW_APP_BASE_URL: ${{ vars.PREVIEW_APP_BASE_URL || 'https://congress-trade-preview.workers.dev' }} + + - name: Deploy to Cloudflare Workers (preview) + id: deploy + run: | + DEPLOY_URL=$(bash scripts/deploy-preview.sh 2>&1 | tee /dev/stderr | grep -o 'https://[^ ]*\.workers\.dev' | tail -1 || true) + echo "deploy_url=${DEPLOY_URL:-${PREVIEW_APP_BASE_URL}}" >> "$GITHUB_OUTPUT" + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + PREVIEW_APP_BASE_URL: ${{ vars.PREVIEW_APP_BASE_URL || 'https://congress-trade-preview.workers.dev' }} + + - name: Comment deploy URL + if: github.event_name == 'push' + run: | + URL="${{ steps.deploy.outputs.deploy_url }}" + PR_NUMBER="$(gh pr list --head staging --json number -q '.[0].number')" + if [ -n "$PR_NUMBER" ]; then + gh pr comment "$PR_NUMBER" --body "Preview deployed: ${URL}" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..b5c959f7d --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,49 @@ +name: Deploy + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type deploy-production to deploy congress.trade" + required: true + type: string + +# Never run two deploys at once; let an in-flight deploy finish rather than +# cancel it mid-deploy. +concurrency: + group: deploy-production + cancel-in-progress: false + +jobs: + deploy: + if: github.event.inputs.confirm == 'deploy-production' + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run typecheck + + - name: Test + run: npm test + + - name: Deploy to Cloudflare Workers + run: bash scripts/ship.sh + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + ADMIN_TOKEN: ${{ secrets.ADMIN_TOKEN }} diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..a158479cc --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,51 @@ +name: Security + +on: + pull_request: + push: + branches: + - main + schedule: + - cron: "41 10 * * 1" + +jobs: + gitleaks: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - name: Refuse untrusted PR source + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "::error::Fork PRs cannot run security scans." + exit 1 + fi + actor="${{ github.actor }}" + if printf '%s' "$actor" | grep -Eq '\[bot\]$'; then + case "$actor" in + 'cursor[bot]'|'dependabot[bot]') + echo "Trusted same-repo bot: $actor" + ;; + *) + echo "::error::Untrusted bot PRs cannot run security scans (untrusted bot: $actor)." + exit 1 + ;; + esac + fi + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Clean stale gitleaks installer temp files + shell: bash + run: | + set -euo pipefail + tmp_root="${TMPDIR:-/tmp}" + tmp_root="${tmp_root%/}" + rm -f "$tmp_root/gitleaks.tmp" + rm -rf "$tmp_root/gitleaks-8.24.3" + - uses: gitleaks/gitleaks-action@e0c47f4f8be36e29cdc102c57e68cb5cbf0e8d1e + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/uptime-monitor.yml b/.github/workflows/uptime-monitor.yml new file mode 100644 index 000000000..6c5a7996f --- /dev/null +++ b/.github/workflows/uptime-monitor.yml @@ -0,0 +1,43 @@ +name: Uptime Monitor + +on: + schedule: + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: {} + +permissions: + contents: read + issues: write + +jobs: + ping: + runs-on: ubuntu-latest + steps: + - name: Health check + id: check + run: | + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" https://congress.trade/health || echo "000") + echo "http_code=$HTTP_CODE" >> "$GITHUB_OUTPUT" + if [ "$HTTP_CODE" != "200" ]; then + echo "status=fail" >> "$GITHUB_OUTPUT" + else + echo "status=ok" >> "$GITHUB_OUTPUT" + fi + + - name: Open issue on failure + if: steps.check.outputs.status == 'fail' + run: | + title="Uptime Alert: congress.trade health returned HTTP ${{ steps.check.outputs.http_code }}" + existing="$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --label incident --search "congress.trade health" --json number -q '.[0].number')" + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" \ + --body "Health check is still failing at $(date -u). HTTP status: ${{ steps.check.outputs.http_code }}." + else + gh issue create \ + --repo "$GITHUB_REPOSITORY" \ + --title "$title" \ + --label "incident" \ + --body "Health check failed at $(date -u). HTTP status: ${{ steps.check.outputs.http_code }}." + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/app/.dev.vars.example b/app/.dev.vars.example index a2096955f..8e0924d49 100644 --- a/app/.dev.vars.example +++ b/app/.dev.vars.example @@ -124,6 +124,10 @@ APP_BASE_URL="" # Throttled to at most one alert per 12h. ALERT_EMAIL="" +# Sentry error monitoring (Cloudflare Workers SDK). +# Get this from Sentry → Projects → congress-trade → Settings → Client Keys (DSN). +SENTRY_DSN="" + # Billing (Stripe) — freemium paywall. Premium unlocks full history + CSV export. # - STRIPE_SECRET_KEY: sk_live_… / sk_test_… (enables billing; absent => 503) # - STRIPE_WEBHOOK_SECRET: whsec_… from the webhook endpoint you create for diff --git a/app/package-lock.json b/app/package-lock.json index 74a54af55..43704248a 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,7 +8,7 @@ "name": "congress-feed", "version": "0.1.0", "dependencies": { - "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", + "@sentry/cloudflare": "^10.62.0", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", @@ -192,7 +192,7 @@ "version": "4.20260625.1", "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260625.1.tgz", "integrity": "sha512-asH0RhPHiNu/IUSssyiOJYAcGqysy0DJpO9fihC6KATaayD9CE1E9bgNQozTLUraxrCT2qkM4CBOIcV0M5NPJw==", - "dev": true, + "devOptional": true, "license": "MIT OR Apache-2.0" }, "node_modules/@cspotcode/source-map-support": { @@ -1174,14 +1174,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@jaywedgeworth22/congress-trading-shared": { - "version": "1.0.0", - "resolved": "git+ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git#c34cfae038ee32d8bb3711fc0365f4f6d4ea585b", - "license": "UNLICENSED", - "dependencies": { - "zod": "^3.23.8" - } - }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1229,6 +1221,15 @@ "@emnapi/runtime": "^1.7.1" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@oxc-project/types": { "version": "0.133.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", @@ -1543,6 +1544,36 @@ "dev": true, "license": "MIT" }, + "node_modules/@sentry/cloudflare": { + "version": "10.62.0", + "resolved": "https://registry.npmjs.org/@sentry/cloudflare/-/cloudflare-10.62.0.tgz", + "integrity": "sha512-oHDpXXiO3XpBO2cHiTRQpSrtQOQrsU9JsO3TZ6ukdd24IUE6Tkc3l7hWdwzKqId3nTWP1Ef0Fr+offsrEGJ6UA==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@sentry/core": "10.62.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.x" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/@sentry/core": { + "version": "10.62.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.62.0.tgz", + "integrity": "sha512-tV69fMg2sS5DUFmQSnS7Jd5qJAp0izxwcsvBVz2ieTM9VMRi99IfOSYW9UYr3p1yfuksk41kefN5PEbeedUE+A==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@sindresorhus/is": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", @@ -3097,6 +3128,7 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/app/package.json b/app/package.json index 821a3beae..4f8cb23aa 100644 --- a/app/package.json +++ b/app/package.json @@ -17,7 +17,7 @@ "preview:deploy": "bash scripts/deploy-preview.sh" }, "dependencies": { - "@jaywedgeworth22/congress-trading-shared": "github:jaywedgeworth22/congress-trading-shared#main", + "@sentry/cloudflare": "^10.62.0", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", diff --git a/app/src/analytics/compute.ts b/app/src/analytics/compute.ts index 8823a6b53..1dd2b74e4 100644 --- a/app/src/analytics/compute.ts +++ b/app/src/analytics/compute.ts @@ -10,16 +10,17 @@ */ import { computePerformance } from '../prices/compute'; -import { bracketMidpoint } from "@jaywedgeworth22/congress-trading-shared"; - -// Re-export for backward compatibility. -export { bracketMidpoint }; /** - * Estimated dollar value of one STOCK Act bracket. Delegate to the shared - * cross-app implementation for consistency with Agentic Trading. + * Estimated dollar value of one STOCK Act bracket. Mirror of + * BRACKET_MIDPOINT_SQL: midpoint of [min,max]; open top tier (max == null) → + * floor (min); missing amount → 0. */ -// bracketMidpoint is now imported from the shared package (above). +export function bracketMidpoint(min: number | null, max: number | null): number { + if (max != null && min != null) return (min + max) / 2; + if (min != null) return min; // open-ended top tier ($50M+) or max missing + return 0; +} /** * Buy/sell sentiment in [0,1]: share of directional (P+S) activity that is diff --git a/app/src/analytics/sql.ts b/app/src/analytics/sql.ts index a39ae365c..a4d76508a 100644 --- a/app/src/analytics/sql.ts +++ b/app/src/analytics/sql.ts @@ -22,10 +22,6 @@ import type { Chamber, TxType } from '../shared/types'; import type { SqlParam } from '../shared/db'; -import type { PartyBucket } from "@jaywedgeworth22/congress-trading-shared"; -import { WINDOW_PRESETS } from "@jaywedgeworth22/congress-trading-shared"; - -export { type PartyBucket, WINDOW_PRESETS }; // --------------------------------------------------------------------------- // Enumerations + validators (closed sets → safe to interpolate as literals) @@ -36,9 +32,7 @@ export { type PartyBucket, WINDOW_PRESETS }; * the presets below, but any positive `d` is valid — so callers can request a * custom age (e.g. ?window=45d) without enumerating it here. */ -// WINDOW_PRESETS is now imported from the shared package. -// Local Window type + validators are kept here (not shared). - +export const WINDOW_PRESETS = ['1d', '7d', '30d', '90d', '180d', '365d', '1825d', 'all'] as const; export type Window = string; // always produced via asWindow(): 'all' | `${number}d` const WINDOW_RE = /^(\d{1,5})d$/; const MAX_WINDOW_DAYS = 36500; // ~100y guardrail against absurd inputs @@ -70,8 +64,7 @@ export function asSourceFilter(v: unknown, fallback: SourceFilter = 'all'): Sour : fallback; } -// PartyBucket is now imported from the shared package and re-exported above. - +export type PartyBucket = 'D' | 'R' | 'O'; export function asPartyBucket(v: unknown): PartyBucket | undefined { if (typeof v !== 'string' || v.length === 0) return undefined; const c = v[0].toUpperCase(); diff --git a/app/src/delivery/__tests__/queueRetry.test.ts b/app/src/delivery/__tests__/queueRetry.test.ts index 34ca5b9e4..0d886be69 100644 --- a/app/src/delivery/__tests__/queueRetry.test.ts +++ b/app/src/delivery/__tests__/queueRetry.test.ts @@ -4,6 +4,14 @@ * webhook retry does not fan out to every subscriber again. */ import { describe, it, expect, vi, afterEach } from 'vitest'; + +// Sentry's queue instrumentation requires AsyncLocalStorage which isn't +// available in vitest. Mock withSentry as a pass-through so tests that call +// worker.queue() directly don't crash on isolation-scope setup. +vi.mock('@sentry/cloudflare', () => ({ + withSentry: (_opts: unknown, handler: unknown) => handler, +})); + import worker from '../../index'; import type { Env, QueueMessage } from '../../shared/types'; diff --git a/app/src/enrichment/compute.ts b/app/src/enrichment/compute.ts index e2de20973..7342855df 100644 --- a/app/src/enrichment/compute.ts +++ b/app/src/enrichment/compute.ts @@ -10,10 +10,17 @@ */ import type { MktCapBucket, SecurityRef } from './types'; -import { marketCapBucket } from "@jaywedgeworth22/congress-trading-shared"; -// Re-export for backward compatibility. -export { marketCapBucket }; +/** Bucket a USD market cap into the standard size tiers. null for missing/≤0. */ +export function marketCapBucket(n: number | null | undefined): MktCapBucket | null { + if (n == null || !Number.isFinite(n) || n <= 0) return null; + if (n >= 200e9) return 'mega'; + if (n >= 10e9) return 'large'; + if (n >= 2e9) return 'mid'; + if (n >= 300e6) return 'small'; + if (n >= 50e6) return 'micro'; + return 'nano'; +} /** * Map a numeric SIC code to a coarse sector via the SEC's SIC division ranges. diff --git a/app/src/enrichment/types.ts b/app/src/enrichment/types.ts index e0a8edf6a..f7aff852e 100644 --- a/app/src/enrichment/types.ts +++ b/app/src/enrichment/types.ts @@ -8,8 +8,36 @@ * fail soft (return null) so a missing key / unknown ticker never throws. */ -import type { MktCapBucket, SecurityRef } from "@jaywedgeworth22/congress-trading-shared"; -export type { MktCapBucket, SecurityRef }; +/** Market-cap size bucket (standard industry thresholds). */ +export type MktCapBucket = 'mega' | 'large' | 'mid' | 'small' | 'micro' | 'nano'; + +/** Reference data for one security, keyed by ticker (mirrors securities_ref). */ +export interface SecurityRef { + ticker: string; + companyName: string | null; + sector: string | null; + industry: string | null; + /** equity | etf | adr | fund | other */ + assetClass: string | null; + isEtf: boolean; + isAdr: boolean; + country: string | null; + stateHq: string | null; + stateOfIncorp: string | null; + exchange: string | null; + exchangeShort: string | null; + currency: string | null; + marketCap: number | null; + marketCapBucket: MktCapBucket | null; + /** Shares outstanding, so market cap can be recomputed from the latest close. */ + sharesOutstanding: number | null; + ipoDate: string | null; + cik: string | null; + sicCode: string | null; + sicDescription: string | null; + /** Which provider(s) produced this row. */ + source: string | null; +} /** * A single enrichment source. `fetchRef` returns the fields it can resolve for a diff --git a/app/src/index.ts b/app/src/index.ts index eb2e32eb2..891863b0b 100644 --- a/app/src/index.ts +++ b/app/src/index.ts @@ -16,6 +16,7 @@ */ import { Hono } from 'hono'; +import * as Sentry from '@sentry/cloudflare'; import type { Env, QueueMessage } from './shared/types'; // Stage handlers owned by their feature modules. @@ -140,45 +141,54 @@ async function handleDeliveryMessage(env: Env, msg: QueueMessage): Promise } } -export default { - /** HTTP entrypoint. */ - fetch(request: Request, env: Env, ctx: ExecutionContext): Promise | Response { - return app.fetch(request, env, ctx); - }, +export default Sentry.withSentry( + (env: Env) => ({ + dsn: env.SENTRY_DSN, + // Send traces for a sample of transactions (0 = off, 1.0 = all). + // Defaults to 0; set to e.g. 0.1 for 10% sampling in production. + tracesSampleRate: 0, + }), + { + /** HTTP entrypoint. */ + fetch(request: Request, env: Env, ctx: ExecutionContext): Promise | Response { + return app.fetch(request, env, ctx); + }, - /** Cron entrypoint — runs every minute; watcher self-gates via shouldPollNow. - * Daily enrichment + price refresh self-gate via a KV date stamp. */ - async scheduled(_event: ScheduledController, env: Env, ctx: ExecutionContext): Promise { - await runWatcher(env, new Date()); - ctx.waitUntil(refreshSecrets(env).catch((err) => console.warn('infisical secret refresh failed:', (err as Error).message))); - ctx.waitUntil(maybeRunDailyJobs(env)); - // Autonomous cross-vendor agreement → auto-publish for a few newly-reviewed - // docs each minute (self-gates on AGREEMENT_AUTOPUBLISH_ENABLED; cron-safe). - ctx.waitUntil( - maybeRunAgreementAutopublish(env).catch((err) => - console.warn('agreement autopublish failed:', (err as Error).message), - ), - ); - }, + /** Cron entrypoint — runs every minute; watcher self-gates via shouldPollNow. + * Daily enrichment + price refresh self-gate via a KV date stamp. */ + async scheduled(_event: ScheduledController, env: Env, ctx: ExecutionContext): Promise { + await runWatcher(env, new Date()); + ctx.waitUntil(refreshSecrets(env).catch((err) => console.warn('infisical secret refresh failed:', (err as Error).message))); + ctx.waitUntil(maybeRunDailyJobs(env)); + // Autonomous cross-vendor agreement → auto-publish for a few newly-reviewed + // docs each minute (self-gates on AGREEMENT_AUTOPUBLISH_ENABLED; cron-safe). + ctx.waitUntil( + maybeRunAgreementAutopublish(env).catch((err) => + console.warn('agreement autopublish failed:', (err as Error).message), + ), + ); + }, - /** - * Queue consumer. Routes by the bound queue name to the ingest/delivery - * handlers. Messages are ack'd individually; failures retry per wrangler.toml. - */ - async queue(batch: MessageBatch, env: Env, _ctx: ExecutionContext): Promise { - const isDelivery = batch.queue.includes('delivery'); - for (const message of batch.messages) { - try { - if (isDelivery) { - await handleDeliveryMessage(env, message.body); - } else { - await handleIngestMessage(env, message.body); + /** + * Queue consumer. Routes by the bound queue name to the ingest/delivery + * handlers. Messages are ack'd individually; failures retry per wrangler.toml. + */ + async queue(batch, env: Env, _ctx: ExecutionContext): Promise { + const isDelivery = batch.queue.includes('delivery'); + for (const message of batch.messages) { + try { + const msg = message.body as QueueMessage; + if (isDelivery) { + await handleDeliveryMessage(env, msg); + } else { + await handleIngestMessage(env, msg); + } + message.ack(); + } catch (err) { + console.error(`queue ${batch.queue} message failed:`, (err as Error).message); + message.retry(); } - message.ack(); - } catch (err) { - console.error(`queue ${batch.queue} message failed:`, (err as Error).message); - message.retry(); } - } + }, }, -}; +); diff --git a/app/src/shared/contracts.ts b/app/src/shared/contracts.ts deleted file mode 100644 index 7357adeea..000000000 --- a/app/src/shared/contracts.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Re-exports from @jaywedgeworth22/congress-trading-shared with local aliases -// for backward compatibility. Prefer importing from the shared package directly -// in new code. - -export { - // Types - type CongressTransaction, - type TransactionsPage, - type TransactionsQuery, - type SecurityRef, - type PriceClose, - type PriceSeries, - type BundleResponse, - type FundamentalRow, - type AnalystRow, - type InsiderRow, - type ShortVolumeRow, - type TickerLeader, - type ClusterBuy, - type MemberLeader, - type MemberPerformance, - type ConvictionTicker, - type BacktestHorizon, - type TickerBacktest, - type CommitteeConflict, - type SharePayload, - type CongressEvent, - type CongressEventType, - type Chamber, - type PartyBucket, - type Owner, - type TxType, - type MktCapBucket, - type SnapshotManifest, - type SnapshotTableInfo, - // Schemas - CongressTransactionSchema, - TransactionsPageSchema, - SecurityRefSchema, - PriceCloseSchema, - SharePayloadSchema, - CongressEventSchema, - ConvictionTickerSchema, - parseArray, - parseSafe, - // Constants - TICKER_ALIASES, - MKT_CAP_THRESHOLDS, - API_PATHS, - WINDOW_PRESETS, - LAG_BUCKETS, - DEFAULT_CONGRESS_TRADE_BASE_URL, - DEFAULT_TRANSACTIONS_LIMIT, - MAX_REFS_BATCH, - APP_B_ORIGIN_TAG, - // Utils - normalizeTicker, - resolveTickerAlias, - marketCapBucket, - bracketMidpoint, - isIsoDate, - daysBetween, - mergeRefs, -} from "@jaywedgeworth22/congress-trading-shared"; diff --git a/app/src/shared/types.ts b/app/src/shared/types.ts index d2b704760..199cf1523 100644 --- a/app/src/shared/types.ts +++ b/app/src/shared/types.ts @@ -7,8 +7,12 @@ */ import type { AssetTypeCategory } from './assetTypes'; -import type { Chamber, Owner, TxType } from "@jaywedgeworth22/congress-trading-shared"; -export type { Chamber, Owner, TxType }; + +// --------------------------------------------------------------------------- +// Primitive unions / enums +// --------------------------------------------------------------------------- + +export type Chamber = 'house' | 'senate'; /** * Filing type code. STOCK Act Periodic Transaction Reports are 'P'. @@ -30,6 +34,12 @@ export type IngestStatus = /** Detected physical form of a disclosure document. */ export type DocKind = 'senate_html' | 'text_pdf' | 'scanned_pdf' | 'unknown'; +/** Beneficial owner of a transaction. */ +export type Owner = 'self' | 'spouse' | 'joint' | 'dependent'; + +/** Transaction type: Purchase | Sale | Exchange. */ +export type TxType = 'P' | 'S' | 'E'; + /** Delivery transport for a subscription. */ export type DeliveryChannel = 'webhook' | 'sse'; @@ -451,6 +461,8 @@ export interface Env { PRICE_PROVIDER?: string; /** HMAC key for signing outbound webhook payloads. */ WEBHOOK_SIGNING_KEY?: string; + /** Sentry DSN for error monitoring (Cloudflare Workers SDK). */ + SENTRY_DSN?: string; // --- End-user auth (public-site sign-in) --- /** Google OAuth client credentials for "Sign in with Google". */ diff --git a/app/src/ui/__tests__/dashboardHtml.test.ts b/app/src/ui/__tests__/dashboardHtml.test.ts index afe798834..652504eb3 100644 --- a/app/src/ui/__tests__/dashboardHtml.test.ts +++ b/app/src/ui/__tests__/dashboardHtml.test.ts @@ -77,6 +77,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('id="colChooser"'); expect(DASHBOARD_HTML).toContain('id="colChooserBody"'); expect(DASHBOARD_HTML).toContain('function resetCols('); + expect(DASHBOARD_HTML).toContain("var COL_ORDER_KEY = 'feed-cols-order-v1'"); + expect(DASHBOARD_HTML).toContain('function moveColumn('); + expect(DASHBOARD_HTML).toContain('Drag columns here to reorder the Trades table.'); + expect(DASHBOARD_HTML).toContain('draggable="true" data-colid'); // the new date/lag columns the user asked for expect(DASHBOARD_HTML).toContain("id: 'traded'"); expect(DASHBOARD_HTML).toContain("id: 'lag'"); @@ -99,6 +103,9 @@ describe('DASHBOARD_HTML', () => { it('keeps account sign-out discoverable from the account menu', () => { expect(DASHBOARD_HTML).toContain('id="acctMenuBtn"'); expect(DASHBOARD_HTML).toContain('Account'); + expect(DASHBOARD_HTML).toContain('themeMenuLabel'); + expect(DASHBOARD_HTML).not.toContain('id="themeToggle"'); + expect(DASHBOARD_HTML).toContain('white-space:nowrap; overflow:hidden; text-overflow:ellipsis;'); expect(DASHBOARD_HTML).toContain('Sign Out'); expect(DASHBOARD_HTML).toContain('function logout()'); }); @@ -186,8 +193,10 @@ describe('DASHBOARD_HTML', () => { it('uses published timing, tighter asset defaults, and source links in drawers', () => { expect(DASHBOARD_HTML).toContain("var sortKey = 'published'"); expect(DASHBOARD_HTML).toContain("var COL_HIDDEN_KEY = 'feed-cols-hidden-v2'"); - expect(DASHBOARD_HTML).toContain("var COL_WIDTH_KEY = 'feed-col-widths-v7'"); - expect(DASHBOARD_HTML).toContain("asset: estimatedColWidth('asset', 54, 48, 62)"); + expect(DASHBOARD_HTML).toContain("var COL_WIDTH_KEY = 'feed-col-widths-v8'"); + expect(DASHBOARD_HTML).toContain("asset: estimatedColWidth('asset', 48, 40, 54)"); + expect(DASHBOARD_HTML).not.toContain('width: max-content'); + expect(DASHBOARD_HTML).not.toContain('feed-col-widths-v7'); expect(DASHBOARD_HTML).toContain('function dateTimeCellHtml('); expect(DASHBOARD_HTML).toContain('date-time-cell'); expect(DASHBOARD_HTML).toContain('#feedTable.resizable th { text-align: center;'); @@ -210,9 +219,15 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('border-right: 1px solid color-mix'); expect(DASHBOARD_HTML).toContain('#feedTable .c-member'); expect(DASHBOARD_HTML).toContain('#feedTable .c-asset'); + expect(DASHBOARD_HTML).toContain(''); + expect(DASHBOARD_HTML).toContain('function syncFeedTableWidth('); + expect(DASHBOARD_HTML).toContain('.clip-text { display:block;'); expect(DASHBOARD_HTML).toContain('drawer-company-title'); expect(DASHBOARD_HTML).toContain('drawer-stack-grid'); expect(DASHBOARD_HTML).toContain('trend-members-grid'); + expect(DASHBOARD_HTML).toContain('.trend-grid2 { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr));'); + expect(DASHBOARD_HTML).toContain('.trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.6fr) minmax(0, .85fr);'); + expect(DASHBOARD_HTML).not.toContain('minmax(260px, .72fr)'); expect(DASHBOARD_HTML).toContain('buySellText('); expect(DASHBOARD_HTML).toContain('0% means matched the S&P'); expect(DASHBOARD_HTML).toContain('Unparsed Historical Filing'); @@ -234,8 +249,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('function useModelRows('); expect(DASHBOARD_HTML).toContain('function openReviewEditor('); expect(DASHBOARD_HTML).toContain('Review / Confirm'); - expect(DASHBOARD_HTML).toContain('Decision History'); - expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=100'"); + expect(DASHBOARD_HTML).toContain('Resolved Reviews'); + expect(DASHBOARD_HTML).toContain('All Filing Decisions'); + expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=200'"); + expect(DASHBOARD_HTML).toContain('function hasAdminToken()'); expect(DASHBOARD_HTML).toContain('function renderDecisionHistory('); expect(DASHBOARD_HTML).toContain('var DECISIONS'); expect(DASHBOARD_HTML).toContain('Use This Model'); @@ -329,6 +346,8 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('class="trend-grid2 timeliness-grid"'); expect(DASHBOARD_HTML).toContain('id="trLagDist" class="lag-dist"'); expect(DASHBOARD_HTML).toContain('class="late-filers-wrap"'); + expect(DASHBOARD_HTML).toContain('.timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(0, .92fr);'); + expect(DASHBOARD_HTML).not.toContain('minmax(280px, .92fr)'); expect(DASHBOARD_HTML).toContain('.late-filers-wrap { max-height: 232px; overflow: auto;'); expect(DASHBOARD_HTML).toContain('Disclosure lag is days between the transaction date and the official filing date.'); expect(DASHBOARD_HTML).toContain('Avg: mean number of days between transaction date and official filing date.'); diff --git a/app/src/ui/dashboardHtml.ts b/app/src/ui/dashboardHtml.ts index 5275b95f2..ed96ff366 100644 --- a/app/src/ui/dashboardHtml.ts +++ b/app/src/ui/dashboardHtml.ts @@ -70,14 +70,12 @@ export const DASHBOARD_HTML = /* html */ ` } html[data-theme="light"] header.top { background: rgba(255,255,255,.72); } /* ---- theme toggle ---- */ - .theme-toggle { background: transparent; border: 1px solid var(--border); color: var(--text-dim); border-radius: 8px; padding: 6px 10px; cursor: pointer; font-size: 13px; line-height: 1; } - .theme-toggle:hover { color: var(--text); background: var(--panel); } /* ---- resizable feed columns ---- */ .table-wrap { overflow-x: auto; max-height: min(78vh, 920px); } - #feedTable.resizable { table-layout: fixed; width: max-content; min-width: 100%; } + #feedTable.resizable { table-layout: fixed; min-width: 100%; } #feedTable.resizable th, #feedTable.resizable td { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } #feedTable.resizable th { text-align: center; padding-right: 18px; } - #feedTable.resizable td > * { max-width: 100%; } + #feedTable.resizable td > * { max-width: 100%; min-width: 0; } #feedTable.resizable .asset-cell, #feedTable.resizable .member-cell { overflow: hidden; max-width: 100%; } #feedTable.resizable .asset-cell > div, @@ -176,6 +174,7 @@ export const DASHBOARD_HTML = /* html */ ` /* let the text shrink inside the (resizable, fixed-layout) cell and clip with an ellipsis instead of wrapping or hard-clipping mid-word */ .asset-cell > div { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .clip-text { display:block; min-width:0; max-width:100%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } .tkr-logo { flex: 0 0 auto; display: inline-flex; align-items: center; justify-content: center; width: 22px; height: 22px; overflow: hidden; } .tkr-logo img { width: 100%; height: 100%; object-fit: contain; display: block; } /* "tile" = frosted-glass box; "transparent" = bare logo on the row surface. */ @@ -330,9 +329,10 @@ export const DASHBOARD_HTML = /* html */ ` .note { font-size:12px; color: var(--text-dim); margin-top:8px; line-height:1.5; } code { font-family: var(--mono); background: var(--bg); padding:1px 6px; border-radius:5px; font-size:12px; color: var(--accent); } /* ================= TRENDS / ANALYTICS ================= */ - .trend-grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; } + .trend-grid2 { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; } + .trend-grid2 > *, .trend-members-grid > *, .trend-side-stack > *, .timeliness-grid > * { min-width: 0; } @media (max-width: 760px) { .trend-grid2 { grid-template-columns: 1fr; } } - .trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.75fr) minmax(260px, .72fr); gap:18px; align-items:start; } + .trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.6fr) minmax(0, .85fr); gap:18px; align-items:start; } .trend-side-stack { display:grid; grid-template-columns:1fr; gap:18px; } @media (max-width: 920px) { .trend-members-grid { grid-template-columns:1fr; } } /* Roomier side drawer on tablets (mobile bottom-sheet still kicks in at 600px). */ @@ -361,7 +361,7 @@ export const DASHBOARD_HTML = /* html */ ` .hfill.buy { background: var(--buy); } .hfill.warn { background: var(--warn); } .hfill.sell { background: var(--sell); } .hbar .hval { width:120px; text-align:right; font-family: var(--mono); font-size:12px; color: var(--text-dim); } .hbar .hval .est-money { font-family: var(--mono); } - .timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(280px, .92fr); align-items: stretch; } + .timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(0, .92fr); align-items: stretch; } .timeliness-panel { min-width: 0; } .timeliness-panel h3 { font-size: 13px; letter-spacing: 0; cursor: help; } .lag-dist { min-height: 232px; display: flex; flex-direction: column; justify-content: space-between; gap: 9px; } @@ -492,11 +492,15 @@ export const DASHBOARD_HTML = /* html */ ` .mini-date { display:flex; flex-direction:column; gap:2px; line-height:1.25; } .mini-date .subline { color:var(--text-dim); font-size:11px; } .mini-source-link { display:block; margin-top:2px; font-size:11px; font-weight:600; } - .colopts { display:flex; flex-wrap:wrap; gap:6px 4px; flex:1; } - .colopt { font-size:13px; color:var(--text); display:inline-flex; align-items:center; gap:5px; margin-right:12px; white-space:nowrap; cursor:pointer; } + .colopts { display:grid; grid-template-columns:repeat(auto-fit,minmax(170px,1fr)); gap:6px; flex:1; } + .colopt { font-size:13px; color:var(--text); display:inline-flex; align-items:center; gap:7px; margin-right:0; white-space:nowrap; cursor:pointer; min-width:0; } button.colopt { font-family:var(--sans); border:1px dashed var(--border); background:color-mix(in srgb,var(--panel-2) 65%,transparent); border-radius:999px; padding:3px 8px; } .colopt.locked { color:var(--text-dim); } .colopt.locked:hover { color:var(--text); border-color:color-mix(in srgb,var(--accent) 55%,var(--border)); } + .colopt.dragging { opacity:.45; border-color:var(--accent); } + .col-drag { color:var(--text-dim); cursor:grab; font-size:14px; line-height:1; } + .colopt input { flex:0 0 auto; } + .colopt-name { overflow:hidden; text-overflow:ellipsis; } .premium-mark { display:inline-flex; align-items:center; justify-content:center; border:1px solid color-mix(in srgb,var(--accent) 42%,var(--border)); background:color-mix(in srgb,var(--accent) 9%,transparent); color:var(--accent); border-radius:999px; padding:1px 6px; font-size:10px; font-weight:800; line-height:1.4; } .panel-note { flex-basis:100%; width:100%; color:var(--text-dim); font-size:12px; line-height:1.45; margin-bottom:4px; } .premium-count-note { margin-left:8px; color:var(--text-dim); } @@ -533,11 +537,11 @@ export const DASHBOARD_HTML = /* html */ ` .acct-menu-btn:hover { background:var(--panel-2); } .acct-menu-btn .acct-caret { color:var(--text-dim); font-size:11px; } .menu { position:relative; } - .menu-pop { position:absolute; right:0; top:38px; background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:6px; min-width:190px; box-shadow:0 12px 32px rgba(0,0,0,.38); display:none; z-index:30; } + .menu-pop { position:absolute; right:0; top:38px; background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:6px; min-width:260px; max-width:min(320px, calc(100vw - 24px)); box-shadow:0 12px 32px rgba(0,0,0,.38); display:none; z-index:30; } .menu-pop.open { display:block; } .menu-pop button { display:block; width:100%; text-align:left; background:transparent; border:none; color:var(--text); padding:8px 10px; border-radius:7px; cursor:pointer; font-size:13px; font-family:var(--sans); } .menu-pop button:hover { background:var(--panel-2); } - .menu-pop .who { padding:6px 10px 8px; font-size:12px; color:var(--text-dim); border-bottom:1px solid var(--border); margin-bottom:5px; word-break:break-all; } + .menu-pop .who { padding:6px 10px 8px; font-size:12px; color:var(--text-dim); border-bottom:1px solid var(--border); margin-bottom:5px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .overlay { position:fixed; inset:0; background:rgba(4,8,16,.62); backdrop-filter:blur(3px); display:none; align-items:center; justify-content:center; z-index:50; padding:18px; } .overlay.open { display:flex; } .modal { background:var(--panel); border:1px solid var(--border); border-radius:16px; padding:26px; width:100%; max-width:430px; box-shadow:0 24px 60px rgba(0,0,0,.45); } @@ -574,7 +578,6 @@ export const DASHBOARD_HTML = /* html */ ` .brand { font-size: 15px; } #srcPill { display: none; } .pill { padding: 3px 7px; } - .theme-toggle { display:none; } nav.tabs { position: fixed; left: 0; right: 0; bottom: 0; margin: 0; width: 100%; max-width: 100%; @@ -1130,10 +1133,9 @@ export const DASHBOARD_HTML = /* html */ ` - +
-
@@ -1180,6 +1182,7 @@ export const DASHBOARD_HTML = /* html */ `
+
@@ -1345,10 +1348,10 @@ export const DASHBOARD_HTML = /* html */ `

Document Review & Model Comparison

-

Scanned / handwritten filings below the confidence threshold are held here until a human acts. Switch to Reviewed to see what was published / rejected / modified, and expand Models on any row to compare each model's confidence and reading.

+

Scanned / handwritten filings below the confidence threshold are held here until a human acts. Switch to Resolved Reviews to see what was published / rejected / modified. The All Filing Decisions table below includes auto-published filings too.

- +
@@ -1356,7 +1359,8 @@ export const DASHBOARD_HTML = /* html */ `
FiledDocStatusReasonPayload

Confirm promotes the read to the live feed; Manual lets you hand-key the rows (recorded as source=manual) when the automated read is wrong or too low-confidence; Reject discards it. Models / readings come from extraction_runs (populated by POST /api/admin/bakeoff). POST /api/admin/review/:docId {decision}

-

Decision History

+

All Filing Decisions

+

Append-only filing decisions, including clean auto-published filings that never entered the review queue.

@@ -1844,6 +1848,11 @@ function esc(s) { }); } function el(id) { return document.getElementById(id); } +function clipTextHtml(value, fallback, title) { + var text = String(value == null || value === '' ? (fallback || '—') : value); + var cls = text === '—' ? 'clip-text muted' : 'clip-text'; + return '' + esc(text) + ''; +} /* Strip stray HTML/entities some upstream datasets embed in asset descriptions (e.g. "
Rate/Coupon: 3.875%
…
"). */ @@ -1886,7 +1895,7 @@ function fmtMs(ms) { function applyTheme(t) { if (t === 'light') document.documentElement.setAttribute('data-theme', 'light'); else document.documentElement.removeAttribute('data-theme'); - var btn = el('themeToggle'); if (btn) btn.textContent = (t === 'light') ? '☀️' : '🌙'; + var label = el('themeMenuLabel'); if (label) label.textContent = (t === 'light') ? 'Light Mode' : 'Dark Mode'; } function toggleTheme() { var cur = document.documentElement.getAttribute('data-theme') === 'light' ? 'light' : 'dark'; @@ -2116,18 +2125,19 @@ var FEED_COLS = [ { id: 'traded', label: 'Traded', sort: 'txdate', def: true, cls: 'muted', tip: 'Date the trade was executed.', cell: function (r) { return dateCellHtml(r.txdate); } }, { id: 'lag', label: 'Lag', sort: 'lag', def: true, tip: 'Days between the trade and the filing (STOCK Act limit: 45).', cell: lagCellHtml }, { id: 'amount', label: 'Amount', sort: 'min', def: true, tip: 'STOCK Act bracket - an estimate, not an exact figure.', cell: amountCellHtml }, - { id: 'sector', label: 'Sector', sort: 'refSector', def: false, cls: 'muted', tier: 'premium', tip: 'Cross-referenced sector (FMP / SEC EDGAR). Blank until the asset is enriched.', cell: function (r) { return r.refSector ? esc(r.refSector) : '—'; } }, - { id: 'marketcap', label: 'Market Cap', sort: 'refMarketCap', def: false, tier: 'premium', tip: 'Market-cap size tier from enriched reference data.', cell: function (r) { return r.refMarketCapBucket ? esc(ownerLabel(r.refMarketCapBucket)) : '—'; } }, - { id: 'country', label: 'Country', sort: 'refCountry', def: false, cls: 'muted', tier: 'premium', tip: 'Country of issue from enriched reference data.', cell: function (r) { return r.refCountry ? esc(r.refCountry) : '—'; } }, - { id: 'owner', label: 'Owner', sort: 'owner', def: false, cls: 'muted', tip: 'Beneficial owner code reported on the filing.', cell: function (r) { return esc(ownerLabel(r.owner) || '—'); } }, + { id: 'sector', label: 'Sector', sort: 'refSector', def: false, cls: 'muted', tier: 'premium', tip: 'Cross-referenced sector (FMP / SEC EDGAR). Blank until the asset is enriched.', cell: function (r) { return clipTextHtml(r.refSector); } }, + { id: 'marketcap', label: 'Market Cap', sort: 'refMarketCap', def: false, tier: 'premium', tip: 'Market-cap size tier from enriched reference data.', cell: function (r) { return clipTextHtml(ownerLabel(r.refMarketCapBucket)); } }, + { id: 'country', label: 'Country', sort: 'refCountry', def: false, cls: 'muted', tier: 'premium', tip: 'Country of issue from enriched reference data.', cell: function (r) { return clipTextHtml(r.refCountry); } }, + { id: 'owner', label: 'Owner', sort: 'owner', def: false, cls: 'muted', tip: 'Beneficial owner code reported on the filing.', cell: function (r) { return clipTextHtml(ownerLabel(r.owner)); } }, { id: 'filed', label: 'Official Filed', sort: 'filed', def: false, cls: 'muted', tip: 'Official disclosure/report date. Historical rows may not include it yet.', cell: filedCellHtml }, { id: 'imported', label: 'Imported', sort: 'imported', def: false, cls: 'muted', tier: 'admin', tip: 'When Congress.Trade imported each filing.', cell: function (r) { return dateTimeCellHtml(r.imported, 'When Congress.Trade imported each filing'); } }, - { id: 'chamber', label: 'Chamber', sort: 'chamber', def: false, cls: 'muted', tip: 'House or Senate source chamber.', cell: function (r) { return esc(ownerLabel(r.chamber) || '—'); } }, + { id: 'chamber', label: 'Chamber', sort: 'chamber', def: false, cls: 'muted', tip: 'House or Senate source chamber.', cell: function (r) { return clipTextHtml(ownerLabel(r.chamber)); } }, { id: 'conf', label: 'Confidence', sort: 'conf', def: false, tier: 'admin', tip: 'Parser confidence after validation penalties.', cell: function (r) { return '~' + (r.conf * 100).toFixed(0) + '%'; } }, - { id: 'source', label: 'Source', sort: 'source', def: false, tier: 'admin', tip: 'Row provenance: primary official pipeline or historical seed import.', cell: function (r) { return '' + esc(sourceLabel(r.source)) + ''; } }, + { id: 'source', label: 'Source', sort: 'source', def: false, tier: 'admin', tip: 'Row provenance: primary official pipeline or historical seed import.', cell: function (r) { return clipTextHtml(sourceLabel(r.source), '—', sourceTitle(r.source)); } }, { id: 'latency', label: 'Latency', sort: null, def: false, cls: 'latency', tier: 'admin', tip: 'Released to seen, then seen to imported for primary rows.', cell: function (r) { return rowLatencyHtml(r); } } ]; var COL_HIDDEN_KEY = 'feed-cols-hidden-v2'; +var COL_ORDER_KEY = 'feed-cols-order-v1'; function isAdminView() { return typeof ME !== 'undefined' && !!(ME.admin && ME.admin.allowed); } @@ -2136,18 +2146,65 @@ function canUseColumn(c) { if (c.tier === 'premium') return isAdminView() || (typeof ME !== 'undefined' && isPremium()); return true; } -function availableCols() { return FEED_COLS.filter(canUseColumn); } +function loadColOrder() { try { var v = JSON.parse(localStorage.getItem(COL_ORDER_KEY)); return Array.isArray(v) ? v : []; } catch (e) { return []; } } +function saveColOrder(v) { try { localStorage.setItem(COL_ORDER_KEY, JSON.stringify(v)); } catch (e) {} } +var colOrder = loadColOrder(); +function orderedCols(cols) { + var pos = {}; + colOrder.forEach(function (id, i) { pos[id] = i; }); + return cols.slice().sort(function (a, b) { + var ai = pos[a.id], bi = pos[b.id]; + if (ai == null && bi == null) return FEED_COLS.indexOf(a) - FEED_COLS.indexOf(b); + if (ai == null) return 1; + if (bi == null) return -1; + return ai - bi; + }); +} +function chooserCols() { + return orderedCols(FEED_COLS.filter(function (c) { + if (c.lock) return false; + if (c.tier === 'admin' && !isAdminView()) return false; + return true; + })); +} +function availableCols() { return orderedCols(FEED_COLS.filter(canUseColumn)); } function defaultHidden() { return availableCols().filter(function (c) { return !c.def; }).map(function (c) { return c.id; }); } function loadHiddenCols() { try { var v = JSON.parse(localStorage.getItem(COL_HIDDEN_KEY)); return v && v.length !== undefined ? v : defaultHidden(); } catch (e) { return defaultHidden(); } } function saveHiddenCols(h) { try { localStorage.setItem(COL_HIDDEN_KEY, JSON.stringify(h)); } catch (e) {} } var hiddenCols = loadHiddenCols(); function isColVisible(id) { return hiddenCols.indexOf(id) < 0; } function visibleCols() { return availableCols().filter(function (c) { return isColVisible(c.id); }); } +function renderFeedColGroup() { + var cg = el('feedCols'); if (!cg) return; + cg.innerHTML = visibleCols().map(function (c) { return ''; }).join(''); +} +function parsePx(v) { + var n = parseFloat(v); + return Number.isFinite(n) ? n : 0; +} +function syncFeedTableWidth() { + var table = el('feedTable'); if (!table) return; + var ths = Array.prototype.slice.call(document.querySelectorAll('#feedHead th')); + var cols = Array.prototype.slice.call(document.querySelectorAll('#feedCols col')); + if (!ths.length) return; + var total = 0; + for (var i = 0; i < ths.length; i++) { + var w = parsePx(ths[i].style.width) || ths[i].offsetWidth || minColWidth(ths[i].dataset.col); + w = Math.max(minColWidth(ths[i].dataset.col), Math.round(w)); + ths[i].style.width = w + 'px'; + if (cols[i]) cols[i].style.width = w + 'px'; + total += w; + } + var wrap = table.closest ? table.closest('.table-wrap') : null; + var min = wrap ? wrap.clientWidth : 0; + table.style.width = Math.max(total, min) + 'px'; +} /* Render the header from the registry, (re)attach sort handlers, and reset the resize state so widths re-freeze for the now-visible columns. */ function renderFeedHeader() { var head = el('feedHead'); if (!head) return; + renderFeedColGroup(); head.innerHTML = visibleCols().map(function (c) { var cls = (c.sort ? 'sortable ' : '') + 'c-' + c.id; var ds = c.sort ? ' data-sort="' + c.sort + '"' : ''; @@ -2157,7 +2214,7 @@ function renderFeedHeader() { var ths = head.querySelectorAll('th.sortable'); for (var i = 0; i < ths.length; i++) { (function (th) { th.onclick = function () { setSort(th.dataset.sort); }; })(ths[i]); } // Re-init the resizable columns for the new header. - var table = el('feedTable'); if (table) table.classList.remove('resizable'); + var table = el('feedTable'); if (table) { table.classList.remove('resizable'); table.style.width = ''; } colResizeInit = false; updateSortIndicators(); } @@ -2190,17 +2247,14 @@ function renderColChooser() { var note = lockedPremium ? '
Premium enrichment
Sector, market cap, and country are available with Premium.
' : ''; - box.innerHTML = note + FEED_COLS.filter(function (c) { - if (c.lock) return false; - if (c.tier === 'admin' && !isAdminView()) return false; - return true; - }).map(function (c) { + note += '
Drag columns here to reorder the Trades table.
'; + box.innerHTML = note + chooserCols().map(function (c) { var tip = c.tip ? ' title="' + esc(c.tip) + '"' : ''; if (c.tier === 'premium' && lockedPremium) { - return ''; + return ''; } - return ''; + return ''; }).join(''); } function toggleColChooser() { @@ -2214,7 +2268,24 @@ function onColToggle(id, visible) { saveHiddenCols(hiddenCols); renderFeedHeader(); renderFeed(); } -function resetCols() { hiddenCols = defaultHidden(); saveHiddenCols(hiddenCols); renderColChooser(); renderFeedHeader(); renderFeed(); } +function moveColumn(dragId, targetId) { + if (!dragId || !targetId || dragId === targetId) return; + var ids = chooserCols().map(function (c) { return c.id; }); + ids = ids.filter(function (id) { return id !== dragId; }); + var idx = ids.indexOf(targetId); + if (idx < 0) return; + ids.splice(idx, 0, dragId); + colOrder = ids; + saveColOrder(colOrder); + renderColChooser(); renderFeedHeader(); renderFeed(); +} +function resetCols() { + hiddenCols = defaultHidden(); + colOrder = []; + saveHiddenCols(hiddenCols); + saveColOrder(colOrder); + renderColChooser(); renderFeedHeader(); renderFeed(); +} function renderFeed() { var m = el('qMember').value.toLowerCase(), t = el('qTicker').value.toUpperCase(), @@ -2251,7 +2322,7 @@ function renderFeed() { if (rows.length === 0) { body.innerHTML = stateRow(cols.length, 'No transactions match these filters.'); if (cards) cards.innerHTML = stateCards('No transactions match these filters.'); - updateFeedCountMsg(0); maybeInitResize(); return; + updateFeedCountMsg(0); maybeInitResize(); syncFeedTableWidth(); return; } body.innerHTML = rows.map(function (r) { var tds = cols.map(function (c) { @@ -2262,6 +2333,7 @@ function renderFeed() { if (cards) cards.innerHTML = rows.map(feedCardHtml).join(''); updateFeedCountMsg(rows.length); maybeInitResize(); + syncFeedTableWidth(); } /* "Showing X-Y of N" + previous/next controls for the bounded table page. */ @@ -2290,7 +2362,7 @@ function updateFeedCountMsg(shown) { } /* ---- resizable feed columns (drag the right edge of a header) ---- */ -var COL_WIDTH_KEY = 'feed-col-widths-v7'; +var COL_WIDTH_KEY = 'feed-col-widths-v8'; var colResizeInit = false; function loadColWidths() { try { return JSON.parse(localStorage.getItem(COL_WIDTH_KEY) || '{}') || {}; } catch (e) { return {}; } } function saveColWidths(w) { try { localStorage.setItem(COL_WIDTH_KEY, JSON.stringify(w)); } catch (e) {} } @@ -2312,7 +2384,7 @@ function estimatedColWidth(key, fallback, min, max) { } function minColWidth(key) { var map = { - asset: 48, + asset: 40, member: 62, amount: 56, imported: 62, @@ -2355,7 +2427,7 @@ function initColumnResize() { // compact default (Asset fits the longest name otherwise) — short entries then // show in full, long ones clip to an ellipsis, and any column stays draggable. var DEFAULT_CAP = { - asset: estimatedColWidth('asset', 54, 48, 62), + asset: estimatedColWidth('asset', 48, 40, 54), member: estimatedColWidth('member', 220, 160, 286) }; for (var i = 0; i < ths.length; i++) { @@ -2366,6 +2438,7 @@ function initColumnResize() { } table.classList.add('resizable'); for (var j = 0; j < ths.length; j++) addColResizer(ths[j]); + syncFeedTableWidth(); applyColumnWidthClasses(); } function addColResizer(th) { @@ -2377,6 +2450,7 @@ function addColResizer(th) { var startX = e.pageX, startW = th.offsetWidth; function move(ev) { th.style.width = Math.max(minColWidth(th.dataset.col), startW + (ev.pageX - startX)) + 'px'; + syncFeedTableWidth(); applyColumnWidthClasses(); } function up() { @@ -2384,6 +2458,7 @@ function addColResizer(th) { document.removeEventListener('mouseup', up); document.body.style.userSelect = ''; var w = loadColWidths(); w[th.dataset.col] = th.offsetWidth; saveColWidths(w); + syncFeedTableWidth(); applyColumnWidthClasses(); } document.addEventListener('mousemove', move); @@ -2736,7 +2811,7 @@ function loadReview() { } function loadDecisionHistory() { // API HOOK: GET /api/admin/ingestion-decisions - return fetch('/api/admin/ingestion-decisions?limit=100', { headers: adminHeaders() }) + return fetch('/api/admin/ingestion-decisions?limit=200', { headers: adminHeaders() }) .then(okOrThrow) .then(function (data) { DECISIONS = data.items || []; @@ -3305,13 +3380,13 @@ function adminHeaders(extra) { } // Turn a 401 into an actionable message instead of a bare "HTTP 401". function adminOk(r) { - if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin access box above.'); + if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin tab access box.'); if (!r.ok) throw new Error('HTTP ' + r.status); return r; } // Like adminOk but only intercepts 401 — lets the caller parse a JSON {error} body for other statuses. function admin401(r) { - if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin access box above.'); + if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin tab access box.'); return r; } function saveAdminToken() { @@ -3319,7 +3394,9 @@ function saveAdminToken() { try { if (v) localStorage.setItem(ADMIN_TOKEN_KEY, v); else localStorage.removeItem(ADMIN_TOKEN_KEY); } catch (e) {} el('adminTokenMsg').textContent = v ? 'Saved in this browser.' : 'Cleared.'; setTimeout(function () { el('adminTokenMsg').textContent = ''; }, 2500); + applyAdminVisibility(); renderFeedHeader(); renderColChooser(); renderFeed(); + if (v) loadReview(); loadPollConfig(); loadHealth(); loadMarketCoverage(); loadDiagnostics(); } function clearAdminToken() { @@ -3327,6 +3404,7 @@ function clearAdminToken() { if (el('adminToken')) el('adminToken').value = ''; el('adminTokenMsg').textContent = 'Cleared.'; setTimeout(function () { el('adminTokenMsg').textContent = ''; }, 2500); + applyAdminVisibility(); renderFeedHeader(); renderColChooser(); renderFeed(); } // Populate the field from storage when the Admin tab opens. @@ -4371,21 +4449,23 @@ function openTrade(row) { ? '' + esc(fmtName(row.member)) + '' : esc(fmtName(row.member)); var sideWord = row.type === 'P' ? 'Bought' : row.type === 'S' ? 'Sold' : 'Exchanged'; + var displayTicker = isScannedPdfPlaceholder(row.ticker) ? '' : (row.ticker || ''); + var displayAsset = cleanAsset(row.asset || ''); // A trade drawer leads with the TRANSACTION (kicker + amount), not the company — // the ticker/company is demoted to a non-clickable "in …" line so it can't be // mistaken for the company drawer (the ticker is intentionally NOT clickable here). - var inName = (row.ticker || row.asset) + var inName = (displayTicker || displayAsset) ? '

in ' + - (row.ticker ? '' + esc(row.ticker) + '' : '') + - (row.ticker && row.asset ? '·' : '') + - (row.asset ? '' + esc(row.asset) + '' : '') + '

' + (displayTicker ? '' + esc(displayTicker) + '' : '') + + (displayTicker && displayAsset ? '·' : '') + + (displayAsset ? '' + esc(displayAsset) + '' : '') + '

' : ''; var personCard = '
Politician
' + memberAvatarHtml(fmtName(row.member), row.photoUrl) + '
' + memberVal + '
'; - var assetLabel = row.asset || row.ticker || 'Asset unavailable'; + var assetLabel = displayAsset || displayTicker || 'Unparsed Historical Filing'; var assetCard = '
Asset
' + - tickerLogoHtml(row.ticker, assetLabel) + '
' + - (row.ticker ? '' + esc(row.ticker) + '' : '') + + tickerLogoHtml(displayTicker, assetLabel) + '
' + + (displayTicker ? '' + esc(displayTicker) + '' : '') + '' + esc(assetLabel) + '
'; var head = '
' + @@ -4454,7 +4534,8 @@ var ME = { user: null, entitlement: { premium: false, status: null, plan: null, var selectedPlan = 'monthly'; function isPremium() { return !!(ME.entitlement && ME.entitlement.premium); } -function canUseAdmin() { return !!(ME.user && ME.admin && ME.admin.allowed); } +function hasAdminToken() { return !!getAdminToken(); } +function canUseAdmin() { return !!((ME.user && ME.admin && ME.admin.allowed) || hasAdminToken()); } function updatePremiumCues() { var unlocked = isPremium() || isAdminView(); document.querySelectorAll('[data-premium-cue]').forEach(function (node) { node.hidden = unlocked; }); @@ -4517,6 +4598,7 @@ function renderAccount() { '' + '
TimeDocActionSourceReasonRows