Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions app/docs/client-mobile-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,10 @@ shared type set but still return `501`.
`chamber`, `type`, `from`, `to`, `order`, and `limit`, and returns the
cursor/count/total metadata used by polling clients.
- Each feed item's `asset` object carries `name` (the disclosed asset text),
`ticker`, `type`, `sector`, and `marketCapBucket`, plus two enrichment fields
shared with the web client so every surface renders identically:
`ticker`, raw disclosure `type`, `typeName`, canonical cross-chamber
`typeCategory` / `typeCategoryLabel`, `sector`, and `marketCapBucket`, plus
two enrichment fields shared with the web client so every surface renders
identically:
- `companyName` — the canonical company name from `securities_ref`
(`null` until the ticker is enriched).
- `logoUrl` — a same-origin path to the cached logo proxy, e.g.
Expand Down
3 changes: 2 additions & 1 deletion app/docs/fmp-data-sharing.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,8 @@ feed rows or public analytics.
Per-transaction object (each item in `transactions[]`):

```
{ id, docId, filerId, txDate, owner, assetName, ticker, assetType, txType,
{ id, docId, filerId, txDate, owner, assetName, ticker, assetType,
assetTypeName, assetTypeCategory, assetTypeCategoryLabel, txType,
amountMin, amountMax, isOption, capGainsOver200, rawText, confidence,
source, createdAt, cursorSeq,
fullName, state, photoUrl, filedDate, firstSeenAt,
Expand Down
5 changes: 4 additions & 1 deletion app/docs/pit-score-export.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@ whether returned rows are safe for historical validation.
Rows are keyed by ticker and market-available disclosure timestamp:

- `observationId`
- `ticker`, `stableSecurityId`, `cusip`, `cik`, `assetType`
- `ticker`, `stableSecurityId`, `cusip`, `cik`
- raw/security asset context: `assetType`, `assetTypeName`
- cross-chamber canonical asset context: `assetTypeCategory`,
`assetTypeCategoryLabel`, `assetTypeCategorySource`
- `tickerMapVersion`, `delistingTickerChangeMetadata`
- `asOf`, `disclosureAvailableAt`, `computedAt`, `dataCutoffAt`
- `scoreVersion`, `parameterManifest`
Expand Down
25 changes: 25 additions & 0 deletions app/migrations/0019_ingestion_decisions.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
-- 0019_ingestion_decisions.sql
-- Append-only audit trail for filing/trade publication decisions. This keeps
-- the review_queue focused on exceptions while preserving every publish/review
-- decision for admin history and future scoring/debugging.

CREATE TABLE IF NOT EXISTS ingestion_decisions (
id TEXT PRIMARY KEY,
doc_id TEXT NOT NULL,
action TEXT NOT NULL,
source TEXT NOT NULL,
actor TEXT,
reason TEXT,
payload TEXT,
transaction_ids TEXT NOT NULL DEFAULT '[]',
created_at TEXT NOT NULL
);

CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_doc
ON ingestion_decisions (doc_id, created_at DESC);

CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_created
ON ingestion_decisions (created_at DESC);

CREATE INDEX IF NOT EXISTS idx_ingestion_decisions_action
ON ingestion_decisions (action, created_at DESC);
70 changes: 70 additions & 0 deletions app/src/admin/__tests__/diagnostics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,33 @@ function fakeDb() {
] as T[],
};
}
if (/FROM securities_ref/i.test(sql) && /CASE\s+WHEN lower\(source\)/i.test(sql)) {
return {
results: [
{
provider: 'massive',
calls_total: 2,
calls_last_24h: 1,
calls_today: 1,
last_used_at: '2026-06-24T11:30:00.000Z',
errors_last_24h: 0,
},
] as T[],
};
}
if (/FROM securities_ref/i.test(sql) && /COUNT\(\*\) AS calls_total/i.test(sql)) {
return {
results: [
{
calls_total: 5,
calls_last_24h: 2,
calls_today: 1,
last_used_at: '2026-06-24T11:00:00.000Z',
errors_last_24h: 1,
},
] as T[],
};
}
if (/FROM securities_ref/i.test(sql) && /enrichment_error/i.test(sql)) {
return {
results: [
Expand All @@ -37,6 +64,42 @@ function fakeDb() {
] as T[],
};
}
if (/FROM price_eod/i.test(sql)) {
return {
results: [
{
calls_total: 7,
calls_last_24h: 20,
calls_today: 3,
last_used_at: '2026-06-24',
},
] as T[],
};
}
if (/FROM spx_eod/i.test(sql)) {
return {
results: [
{
calls_total: 100,
calls_last_24h: 1,
calls_today: 1,
last_used_at: '2026-06-24',
},
] as T[],
};
}
if (/FROM tx_performance/i.test(sql)) {
return {
results: [
{
calls_total: 50,
calls_last_24h: 10,
calls_today: 5,
last_used_at: '2026-06-24T12:30:00.000Z',
},
] as T[],
};
}
if (/FROM deliveries/i.test(sql)) return { results: [] as T[] };
if (/FROM review_queue/i.test(sql)) return { results: [] as T[] };
if (/FROM client_commands/i.test(sql)) return { results: [] as T[] };
Expand Down Expand Up @@ -72,6 +135,8 @@ describe('admin diagnostics API', () => {
ADMIN_TOKEN: 'admin-secret',
GEMINI_API_KEY: 'gemini-secret',
FMP_API_KEY: 'fmp-secret',
MASSIVE_API_KEY: 'massive-secret',
PRICE_PROVIDER: 'massive',
DB: fakeDb(),
} as never,
);
Expand All @@ -91,10 +156,15 @@ describe('admin diagnostics API', () => {
callsToday: 1,
}),
expect.objectContaining({ id: 'source:house', status: 'ok', callsToday: 2 }),
expect.objectContaining({ id: 'provider:massive', status: 'ok', configured: true, callsToday: 1 }),
expect.objectContaining({ id: 'cache:prices', status: 'ok', configured: true, callsToday: 3 }),
expect.objectContaining({ id: 'cache:spx', status: 'ok', configured: true, callsToday: 1 }),
expect.objectContaining({ id: 'cache:performance', status: 'ok', configured: true, callsToday: 5 }),
]),
);
expect(JSON.stringify(body)).not.toContain('gemini-secret');
expect(JSON.stringify(body)).not.toContain('fmp-secret');
expect(JSON.stringify(body)).not.toContain('massive-secret');
expect(body.errors).toEqual(
expect.arrayContaining([
expect.objectContaining({
Expand Down
46 changes: 46 additions & 0 deletions app/src/admin/__tests__/reviewQueue.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ describe('review queue admin API', () => {
'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf',
raw_object_key: 'raw/H-2026-2003695',
doc_kind: 'scanned_pdf',
chamber: 'house',
},
]),
} as never,
Expand All @@ -55,6 +56,7 @@ describe('review queue admin API', () => {
sourceUrl: string;
rawObjectKey: string;
docKind: string;
chamber: string;
payload: { minConfidence: number; transactions: unknown[] };
}>;
};
Expand All @@ -63,6 +65,7 @@ describe('review queue admin API', () => {
sourceUrl: 'https://disclosures-clerk.house.gov/public_disc/ptr-pdfs/2026/2003695.pdf',
rawObjectKey: 'raw/H-2026-2003695',
docKind: 'scanned_pdf',
chamber: 'house',
payload: { minConfidence: 0, transactions: [] },
});
});
Expand Down Expand Up @@ -97,6 +100,44 @@ describe('review queue admin API', () => {
expect(body.items[0]).toMatchObject({ resolved: true, ingestStatus: 'persisted' });
});

it('lists ingestion decision history separately from the review queue', async () => {
const res = await app.request(
'/ingestion-decisions',
{ headers: { Authorization: 'Bearer admin-secret' } },
{
ADMIN_TOKEN: 'admin-secret',
DB: fakeDb([
{
id: 'dec-1',
doc_id: 'S-1',
action: 'auto_published',
source: 'pipeline',
actor: null,
reason: 'passed_normalization',
payload: '{"inserted":2}',
transaction_ids: '["tx1","tx2"]',
created_at: '2026-06-29T00:00:00.000Z',
chamber: 'senate',
ingest_status: 'persisted',
source_url: 'https://example/senate',
},
]),
} as never,
);
expect(res.status).toBe(200);
const body = (await res.json()) as {
available: boolean;
items: Array<{ docId: string; action: string; payload: { inserted: number }; transactionIds: string[] }>;
};
expect(body.available).toBe(true);
expect(body.items[0]).toMatchObject({
docId: 'S-1',
action: 'auto_published',
payload: { inserted: 2 },
transactionIds: ['tx1', 'tx2'],
});
});

it('unpublishes a persisted filing: soft-deletes rows, reverts, re-opens review', async () => {
// fakeDb whose filing lookup resolves and whose UPDATE reports 3 retracted rows.
const db = {
Expand Down Expand Up @@ -214,12 +255,14 @@ describe('review queue admin API', () => {
it("decision='manual' records hand-entered rows as source='manual'", async () => {
// Capture the INSERT bind params so we can assert the source column = 'manual'.
const binds: unknown[][] = [];
const auditBinds: unknown[][] = [];
const db = {
prepare(sql: string) {
return {
_sql: sql,
bind(...args: unknown[]) {
if (/INSERT OR IGNORE INTO transactions/.test(sql)) binds.push(args);
if (/INSERT INTO ingestion_decisions/.test(sql)) auditBinds.push(args);
return this;
},
async all<T>() {
Expand Down Expand Up @@ -256,5 +299,8 @@ describe('review queue admin API', () => {
// The transactions INSERT bound source='manual' (it's the 17th positional bind).
expect(binds.length).toBe(1);
expect(binds[0]).toContain('manual');
expect(auditBinds.length).toBe(1);
expect(auditBinds[0]).toContain('manual');
expect(auditBinds[0]).toContain('admin-token');
});
});
Loading
Loading