Expand disclosure latency providers

app/.dev.vars.example

@@ -86,11 +86,19 @@ IMPORT_MAX_CLOSES_PER_TICKER="1500"
 IMPORT_MAX_INSIDER="5000"
 IMPORT_MAX_SHORT_VOLUME="5000"
 
-# Optional temporary benchmark: polls FMP's House/Senate latest-disclosure
-# endpoints and records whether Congress.Trade saw new filings before FMP did.
+# Optional disclosure-latency benchmark: polls third-party latest-disclosure
+# endpoints and records whether Congress.Trade saw new filings first.
+# Supported direct providers: fmp, unusual_whales, quiver. Finnhub and AInvest
+# are symbol-scoped and Capitol Trades has no stable server API, so those are
+# reported in admin provider status but not automatically probed.
 # Turn on before waiting for the next new disclosures; default code path is off.
+DISCLOSURE_LATENCY_WATCH_ENABLED=""
+DISCLOSURE_LATENCY_PROVIDERS="fmp,unusual_whales,quiver"
+DISCLOSURE_LATENCY_WATCH_LIMIT="100"
 FMP_DISCLOSURE_WATCH_ENABLED=""
 FMP_DISCLOSURE_WATCH_LIMIT="100"
+# Optional provider credentials are resolved from .dev.vars or Worker secrets
+# when those providers are enabled: Unusual Whales, Quiver, and AInvest.
 
 # Cloudflare Access (Zero Trust) sign-in for humans — an ALTERNATIVE/ADDITION to
 # the bearer token. With these set AND an Access application fronting

app/migrations/0021_disclosure_latency_watch.sql

@@ -1,8 +1,8 @@
 -- 0021_disclosure_latency_watch.sql
--- Records Congress.Trade-vs-FMP disclosure discovery timing. Candidates are
+-- Records Congress.Trade-vs-provider disclosure discovery timing. Candidates are
 -- created when our watcher first sees a new filing; provider observations are
--- populated from FMP latest endpoints so we can tell whether FMP was already
--- aware or caught up later.
+-- populated from provider latest endpoints so we can tell who was already aware
+-- or caught up later.
 
 CREATE TABLE IF NOT EXISTS disclosure_latency_candidates (
   doc_id                 TEXT NOT NULL,

app/migrations/0023_disclosure_provider_timestamps.sql

@@ -0,0 +1,6 @@
+-- 0023_disclosure_provider_timestamps.sql
+-- Stores provider-supplied publication/upload timestamps separately from the
+-- monitor's first-observed timestamp. Not every provider exposes this.
+
+ALTER TABLE disclosure_latency_candidates ADD COLUMN provider_published_at TEXT;
+ALTER TABLE disclosure_provider_observations ADD COLUMN provider_published_at TEXT;

app/src/admin/__tests__/disclosureLatency.test.ts

@@ -26,6 +26,7 @@ function fakeDb() {
                   congress_first_seen_at: '2026-06-29T14:00:00.000Z',
                   provider_key: '20012345',
                   provider_first_seen_at: '2026-06-29T14:03:30.000Z',
+                  provider_published_at: '2026-06-29T14:02:00.000Z',
                   match_method: 'doc-token',
                   status: 'matched',
                   attempts: 2,
@@ -59,13 +60,36 @@ describe('admin disclosure latency API', () => {
     );
 
     expect(res.status).toBe(200);
-    const body = (await res.json()) as { items: Array<{ docId: string; providerDeltaSec: number; status: string }> };
+    const body = (await res.json()) as {
+      items: Array<{ docId: string; providerDeltaSec: number; providerPublishedDeltaSec: number; status: string }>;
+    };
     expect(body.items).toEqual([
       expect.objectContaining({
         docId: 'H-2026-20012345',
         providerDeltaSec: 210,
+        providerPublishedDeltaSec: 120,
         status: 'matched',
       }),
     ]);
   });
+
+  it('returns aggregate metrics and a public-safe summary payload', async () => {
+    const res = await app.request(
+      '/disclosure-latency/summary',
+      { headers: { Authorization: 'Bearer admin-secret' } },
+      { ADMIN_TOKEN: 'admin-secret', FMP_API_KEY: 'configured', DB: fakeDb() } as never,
+    );
+
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as {
+      totals: { candidates: number; matched: number; configuredComparableProviders: number };
+      providers: Array<{ provider: string; avgMonitorDeltaSec: number | null; avgProviderPublishedDeltaSec: number | null }>;
+      publicSummary: { providers: Array<{ provider: string }> };
+    };
+    expect(body.totals).toEqual(expect.objectContaining({ candidates: 1, matched: 1, configuredComparableProviders: 1 }));
+    expect(body.providers[0]).toEqual(
+      expect.objectContaining({ provider: 'fmp', avgMonitorDeltaSec: 210, avgProviderPublishedDeltaSec: 120 }),
+    );
+    expect(JSON.stringify(body.publicSummary)).not.toContain('H-2026-20012345');
+  });
 });

app/src/admin/routes.ts

@@ -74,7 +74,7 @@ import { mergeRefs } from '../enrichment/compute';
 import type { SecurityRef } from '../enrichment/types';
 import { runPriceRefresh } from '../prices/service';
 import { getSecretResolverStatus, refreshSecrets, resolveSecret, resolveSecrets } from '../secrets/infisical';
-import { runFmpDisclosureLatencyProbe } from '../ingestion/fmpDisclosureLatency';
+import { getDisclosureLatencySummary, runDisclosureLatencyProbe } from '../ingestion/fmpDisclosureLatency';
 
 // Optional secrets/vars; not declared on Env (frozen). Read defensively.
 type EnvWithAdmin = Env & {
@@ -1131,12 +1131,22 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
     return c.json({ ok: true, latencyResetAt });
   });
 
+  // --- GET /disclosure-latency/summary ------------------------------------
+  // Aggregate provider-race metrics. `publicSummary` intentionally excludes
+  // filing/member detail so it can be reviewed before any public sharing.
+  r.get('/disclosure-latency/summary', async (c) => {
+    return c.json(await getDisclosureLatencySummary(c.env));
+  });
+
   // --- GET /disclosure-latency -------------------------------------------
-  // Congress.Trade-vs-FMP race monitor. `providerDeltaSec` is FMP monitor
-  // first-observed minus Congress.Trade first_seen_at: positive means we observed
-  // first; negative means FMP was already observed first.
+  // Congress.Trade-vs-provider race monitor. `providerDeltaSec` is provider
+  // monitor first-observed minus Congress.Trade first_seen_at: positive means we
+  // observed first; negative means the provider was already observed first.
   r.get('/disclosure-latency', async (c) => {
     const limit = Math.min(Math.max(parseInt(c.req.query('limit') || '50', 10) || 50, 1), 200);
+    const provider = (c.req.query('provider') || '').trim().toLowerCase();
+    const where = provider ? 'WHERE provider = ?' : '';
+    const params: SqlParam[] = provider ? [provider, limit] : [limit];
     const rows = await optionalAll<{
       doc_id: string;
       provider: string;
@@ -1147,6 +1157,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
       congress_first_seen_at: string;
       provider_key: string | null;
       provider_first_seen_at: string | null;
+      provider_published_at: string | null;
       match_method: string | null;
       status: string;
       attempts: number;
@@ -1157,13 +1168,14 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
     }>(
       c.env,
       `SELECT doc_id, provider, chamber, source_url, filed_date, filer_name,
-              congress_first_seen_at, provider_key, provider_first_seen_at,
+              congress_first_seen_at, provider_key, provider_first_seen_at, provider_published_at,
               match_method, status, attempts, last_checked_at, error,
               created_at, updated_at
          FROM disclosure_latency_candidates
+        ${where}
         ORDER BY created_at DESC
         LIMIT ?`,
-      [limit],
+      params,
     );
     const items = rows.map((row) => ({
       docId: row.doc_id,
@@ -1176,6 +1188,8 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
       providerKey: row.provider_key,
       providerFirstSeenAt: row.provider_first_seen_at,
       providerDeltaSec: deltaSeconds(row.provider_first_seen_at, row.congress_first_seen_at),
+      providerPublishedAt: row.provider_published_at,
+      providerPublishedDeltaSec: deltaSeconds(row.provider_published_at, row.congress_first_seen_at),
       matchMethod: row.match_method,
       status: row.status,
       attempts: row.attempts,
@@ -1188,10 +1202,15 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
   });
 
   // --- POST /disclosure-latency/probe -------------------------------------
-  // Force a one-off FMP latest probe, useful immediately after new filings land
-  // or before turning on the continuous cron switch.
+  // Force a one-off provider latest probe, useful immediately after new filings
+  // land or before turning on the continuous cron switch. Optional query:
+  // ?providers=fmp,unusual_whales,quiver
   r.post('/disclosure-latency/probe', async (c) => {
-    const result = await runFmpDisclosureLatencyProbe(c.env, new Date(), fetch, { force: true });
+    const providers = (c.req.query('providers') || c.req.query('provider') || '')
+      .split(/[,\s]+/)
+      .map((part) => part.trim())
+      .filter(Boolean);
+    const result = await runDisclosureLatencyProbe(c.env, new Date(), fetch, { force: true, providers });
     return c.json({ ok: result.errors.length === 0, ...result });
   });
 
@@ -2737,6 +2756,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
          congress_first_seen_at TEXT NOT NULL,
          provider_key TEXT,
          provider_first_seen_at TEXT,
+         provider_published_at TEXT,
          match_method TEXT,
          status TEXT NOT NULL DEFAULT 'pending',
          attempts INTEGER NOT NULL DEFAULT 0,
@@ -2755,6 +2775,7 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
          provider_key TEXT NOT NULL,
          first_observed_at TEXT NOT NULL,
          last_observed_at TEXT NOT NULL,
+         provider_published_at TEXT,
          source_url TEXT,
          filed_date TEXT,
          filer_name TEXT,
@@ -2772,6 +2793,9 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> {
        )`,
       `CREATE INDEX IF NOT EXISTS idx_stripe_webhook_events_received
          ON stripe_webhook_events (received_at DESC)`,
+      // 0023_disclosure_provider_timestamps.sql — provider-side publish/upload timestamp when available.
+      'ALTER TABLE disclosure_latency_candidates ADD COLUMN provider_published_at TEXT',
+      'ALTER TABLE disclosure_provider_observations ADD COLUMN provider_published_at TEXT',
     ];
     const applied: string[] = [];
     const skipped: string[] = [];

app/src/index.ts

@@ -36,7 +36,7 @@ import { buildUiRouter } from './ui/routes';
 import { maybeRunDailyJobs } from './jobs';
 import { maybeRunAgreementAutopublish, handleAgreementCheck } from './extraction/agreement';
 import { refreshSecrets } from './secrets/infisical';
-import { runFmpDisclosureLatencyProbe } from './ingestion/fmpDisclosureLatency';
+import { runDisclosureLatencyProbe } from './ingestion/fmpDisclosureLatency';
 
 const app = new Hono<{ Bindings: Env }>();
 
@@ -161,8 +161,8 @@ export default Sentry.withSentry(
       await runWatcher(env, new Date());
       ctx.waitUntil(refreshSecrets(env).catch((err) => console.warn('infisical secret refresh failed:', (err as Error).message)));
       ctx.waitUntil(
-        runFmpDisclosureLatencyProbe(env).catch((err) =>
-          console.warn('fmp disclosure latency probe failed:', (err as Error).message),
+        runDisclosureLatencyProbe(env).catch((err) =>
+          console.warn('disclosure latency probe failed:', (err as Error).message),
         ),
       );
       ctx.waitUntil(maybeRunDailyJobs(env));

app/src/ingestion/__tests__/fmpDisclosureLatency.test.ts

@@ -1,5 +1,11 @@
 import { describe, expect, it } from 'vitest';
-import { matchFmpDisclosureCandidate, parseFmpDisclosureRows } from '../fmpDisclosureLatency';
+import {
+  matchDisclosureCandidate,
+  matchFmpDisclosureCandidate,
+  parseFmpDisclosureRows,
+  parseQuiverDisclosureRows,
+  parseUnusualWhalesDisclosureRows,
+} from '../fmpDisclosureLatency';
 
 describe('parseFmpDisclosureRows', () => {
   it('extracts a House doc token from PTR PDF URLs', () => {
@@ -86,6 +92,65 @@ describe('matchFmpDisclosureCandidate', () => {
         },
         row,
       ),
-    ).toEqual({ providerKey: row.providerKey, matchMethod: 'probable-filer-date' });
+    ).toEqual({ providerKey: row.providerKey, matchMethod: 'filer-date' });
+  });
+});
+
+describe('parse third-party disclosure providers', () => {
+  it('normalizes Unusual Whales recent Congress rows', () => {
+    const rows = parseUnusualWhalesDisclosureRows({
+      data: [
+        {
+          filed_at_date: '2026-06-29',
+          member_type: 'senate',
+          name: 'Jane Smith',
+          politician_id: 'abc',
+          ticker: 'MSFT',
+          transaction_date: '2026-06-20',
+          txn_type: 'Buy',
+        },
+      ],
+    });
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toEqual(
+      expect.objectContaining({
+        provider: 'unusual_whales',
+        chamber: 'senate',
+        filedDate: '2026-06-29',
+        filerName: 'Jane Smith',
+        providerPublishedAt: null,
+      }),
+    );
+    expect(
+      matchDisclosureCandidate(
+        { doc_id: 'S-hidden', source_url: null, filed_date: '2026-06-29', filer_name: 'Smith, Jane' },
+        rows[0],
+      ),
+    ).toEqual({ providerKey: rows[0].providerKey, matchMethod: 'filer-date' });
+  });
+
+  it('captures Quiver upload timestamps separately from monitor observation time', () => {
+    const rows = parseQuiverDisclosureRows('house', [
+      {
+        Representative: 'Jane Smith',
+        ReportDate: '2026-06-29T00:00:00Z',
+        Date: '2026-06-20T00:00:00Z',
+        Ticker: 'MSFT',
+        Transaction: 'Purchase',
+        Quiver_Upload_Time: '2026-06-29T14:05:00Z',
+      },
+    ]);
+
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toEqual(
+      expect.objectContaining({
+        provider: 'quiver',
+        chamber: 'house',
+        filedDate: '2026-06-29',
+        filerName: 'Jane Smith',
+        providerPublishedAt: '2026-06-29T14:05:00.000Z',
+      }),
+    );
   });
 });