diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4381c8769 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +version: 2 +updates: + - package-ecosystem: npm + directory: "/app" + schedule: + interval: daily + time: "08:00" + open-pull-requests-limit: 10 + groups: + cloudflare: + patterns: + - "@cloudflare/*" + - "wrangler" + - "hono" + testing: + patterns: + - "vitest" + - "@playwright/*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6932585d5..dc78899c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,10 +22,35 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: app/package-lock.json + - name: Configure private GitHub dependencies + env: + SHARED_PACKAGE_DEPLOY_KEY: ${{ secrets.SHARED_PACKAGE_DEPLOY_KEY }} + GH_PAT: ${{ secrets.GH_PAT }} + run: | + if [ -n "$SHARED_PACKAGE_DEPLOY_KEY" ]; then + mkdir -p "$HOME/.ssh" + printf '%s\n' "$SHARED_PACKAGE_DEPLOY_KEY" > "$HOME/.ssh/congress_trading_shared" + chmod 600 "$HOME/.ssh/congress_trading_shared" + ssh-keyscan github.com >> "$HOME/.ssh/known_hosts" + git config --global core.sshCommand "ssh -i $HOME/.ssh/congress_trading_shared -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" + git config --global url."ssh://git@github.com/".insteadOf "git@github.com:" + git ls-remote ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + elif [ -n "$GH_PAT" ]; then + git config --global credential.helper store + printf 'https://x-access-token:%s@github.com\n' "$GH_PAT" > "$HOME/.git-credentials" + git config --global url."https://github.com/".insteadOf "ssh://git@github.com/" + git config --global url."https://github.com/".insteadOf "git@github.com:" + git ls-remote https://github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + else + echo "::error::SHARED_PACKAGE_DEPLOY_KEY or GH_PAT with read access to jaywedgeworth22/congress-trading-shared is required for npm ci." + exit 1 + fi + - name: Install dependencies run: npm ci @@ -34,3 +59,6 @@ jobs: - name: Test run: npm test + + - name: Audit + run: npm audit diff --git a/.github/workflows/codex-autofix.yml b/.github/workflows/codex-autofix.yml index f4db50ddd..389a69931 100644 --- a/.github/workflows/codex-autofix.yml +++ b/.github/workflows/codex-autofix.yml @@ -1,6 +1,8 @@ name: Codex Autofix # Autonomous responder to the Codex PR reviewer (chatgpt-codex-connector[bot]). +# Calls the shared reusable workflow in congress-trading-shared; the prompt +# (repo-specific behaviour) stays here so it's auditable in-repo. # # Roles are DISTINCT so the two bots never compete to "review first": # • Codex = reviewer — fires on every push, posts P1/P2 suggestions. @@ -8,7 +10,7 @@ name: Codex Autofix # then pushes. The push makes Codex review again → # clean ping-pong, capped to avoid an infinite loop. # -# Prerequisites (one-time, see the PR description): +# Prerequisites: # 1. Secret ANTHROPIC_API_KEY (Settings → Secrets and variables → Actions). # 2. A token whose pushes RE-TRIGGER CI + Codex. The default GITHUB_TOKEN does # NOT re-trigger workflows, so EITHER install the Claude GitHub App @@ -40,65 +42,51 @@ concurrency: jobs: autofix: - # Only when the Codex bot posted the feedback (review / inline comment / PR - # comment), or a maintainer dispatched it manually. issue_comment must be on - # a PR (issue.pull_request != null), never a plain issue. if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_review' && github.event.review.user.login == 'chatgpt-codex-connector[bot]') || (github.event_name == 'pull_request_review_comment' && github.event.comment.user.login == 'chatgpt-codex-connector[bot]') || (github.event_name == 'issue_comment' && github.event.issue.pull_request != null && github.event.comment.user.login == 'chatgpt-codex-connector[bot]') - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }} + uses: jaywedgeworth22/congress-trading-shared/.github/workflows/codex-autofix-reusable.yml@main + with: + allowed_bots: "chatgpt-codex-connector,chatgpt-codex-connector[bot]" + prompt: | + You are the autonomous fixer that responds to the Codex PR reviewer + (chatgpt-codex-connector[bot]) on THIS pull request. You do NOT review + the PR yourself — Codex is the reviewer; you only address its feedback. - - uses: anthropics/claude-code-action@v1 - with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ secrets.GH_PAT || secrets.GITHUB_TOKEN }} - # The whole policy lives in the prompt so behavior is auditable in-repo. - prompt: | - You are the autonomous fixer that responds to the Codex PR reviewer - (chatgpt-codex-connector[bot]) on THIS pull request. You do NOT review - the PR yourself — Codex is the reviewer; you only address its feedback. + Repo: Congress.Trade (Cloudflare Worker; the app lives in `app/`). + Verify before committing: `cd app && npm run typecheck && npm test`. + Read app/AGENTS.md and app/CLAUDE.md first — follow them as the source + of truth (migrations, deploy gates, do-not-deploy rules). The git + author email MUST be 12656028+jaywedgeworth22@users.noreply.github.com. + Do NOT deploy, run remote D1 migrations, or run production crawlers. - Repo: Congress.Trade (Cloudflare Worker; the app lives in `app/`). - Verify before committing: `cd app && npm run typecheck && npm test`. - Read app/AGENTS.md and app/CLAUDE.md first — follow them as the source - of truth (migrations, deploy gates, do-not-deploy rules). The git - author email MUST be 12656028+jaywedgeworth22@users.noreply.github.com. - Do NOT deploy, run remote D1 migrations, or run production crawlers. - - Do this: - 1. ROUND CAP: count commits on the PR branch whose message contains - "[codex-autofix]". If there are already 10 or more, STOP: post one PR - comment summarizing the remaining open Codex items and asking the - maintainer how to proceed, then end without further changes. - 2. Read the PR's review threads. Separate OUTDATED threads (anchored to - code already changed — usually already fixed) from genuinely NEW, - non-outdated Codex items. - 3. For each NEW item: if it is a clear correctness bug OR a simple - cosmetic/doc fix, fix it. If it is ambiguous or architecturally - significant, do NOT guess — post a PR comment asking the maintainer, - and skip it. - 4. If `git merge origin/main` is needed (branch behind main), merge it - and resolve conflicts. If a migration is added, add SQL under - app/migrations/ and update POST /api/admin/migrate per app/AGENTS.md. - 5. Run `cd app && npm run typecheck && npm test`. Only commit if it - passes. Commit message must start with "[codex-autofix] ". Push to - the PR branch. - 6. When the PR is functional and you have addressed the actionable - items, ensure auto-merge is enabled: - `gh pr merge --squash --auto`. Do NOT use --admin and do NOT - try to bypass any required check. - 7. Be frugal with PR comments — only comment to ask the maintainer a - question, to report the round cap was hit, or to flag a finding you - are intentionally not fixing. The diff is the record otherwise. - claude_args: | - --max-turns 60 - --allowedTools "Edit,Write,Read,Bash" - allowed_bots: "chatgpt-codex-connector,chatgpt-codex-connector[bot]" + Do this: + 1. ROUND CAP: count commits on the PR branch whose message contains + "[codex-autofix]". If there are already 10 or more, STOP: post one PR + comment summarizing the remaining open Codex items and asking the + maintainer how to proceed, then end without further changes. + 2. Read the PR's review threads. Separate OUTDATED threads (anchored to + code already changed — usually already fixed) from genuinely NEW, + non-outdated Codex items. + 3. For each NEW item: if it is a clear correctness bug OR a simple + cosmetic/doc fix, fix it. If it is ambiguous or architecturally + significant, do NOT guess — post a PR comment asking the maintainer, + and skip it. + 4. If `git merge origin/main` is needed (branch behind main), merge it + and resolve conflicts. If a migration is added, add SQL under + app/migrations/ and update POST /api/admin/migrate per app/AGENTS.md. + 5. Run `cd app && npm run typecheck && npm test`. Only commit if it + passes. Commit message must start with "[codex-autofix] ". Push to + the PR branch. + 6. When the PR is functional and you have addressed the actionable + items, ensure auto-merge is enabled: + `gh pr merge --squash --auto`. Do NOT use --admin and do NOT + try to bypass any required check. + 7. Be frugal with PR comments — only comment to ask the maintainer a + question, to report the round cap was hit, or to flag a finding you + are intentionally not fixing. The diff is the record otherwise. + secrets: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GH_PAT: ${{ secrets.GH_PAT }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 000000000..cfecd4f94 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,102 @@ +name: Deploy Preview + +on: + push: + branches: [staging] + workflow_dispatch: {} + +# Never run two preview deploys at once; let an in-flight deploy finish. +concurrency: + group: deploy-preview + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + pull-requests: write + defaults: + run: + working-directory: app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: app/package-lock.json + + - name: Configure private GitHub dependencies + env: + SHARED_PACKAGE_DEPLOY_KEY: ${{ secrets.SHARED_PACKAGE_DEPLOY_KEY }} + GH_PAT: ${{ secrets.GH_PAT }} + run: | + if [ -n "$SHARED_PACKAGE_DEPLOY_KEY" ]; then + mkdir -p "$HOME/.ssh" + printf '%s\n' "$SHARED_PACKAGE_DEPLOY_KEY" > "$HOME/.ssh/congress_trading_shared" + chmod 600 "$HOME/.ssh/congress_trading_shared" + ssh-keyscan github.com >> "$HOME/.ssh/known_hosts" + git config --global core.sshCommand "ssh -i $HOME/.ssh/congress_trading_shared -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" + git config --global url."ssh://git@github.com/".insteadOf "git@github.com:" + git ls-remote ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + elif [ -n "$GH_PAT" ]; then + git config --global credential.helper store + printf 'https://x-access-token:%s@github.com\n' "$GH_PAT" > "$HOME/.git-credentials" + git config --global url."https://github.com/".insteadOf "ssh://git@github.com/" + git config --global url."https://github.com/".insteadOf "git@github.com:" + git ls-remote https://github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + else + echo "::error::SHARED_PACKAGE_DEPLOY_KEY or GH_PAT with read access to jaywedgeworth22/congress-trading-shared is required for npm ci." + exit 1 + fi + + - name: Install dependencies + run: npm ci + + - name: Build isolated preview config + run: | + cp wrangler.preview.example.toml wrangler.preview.toml + python - <<'PY' + from pathlib import Path + import os + + path = Path("wrangler.preview.toml") + text = path.read_text() + replacements = { + "PREVIEW_D1_DATABASE_ID": os.environ["PREVIEW_D1_DATABASE_ID"], + "PREVIEW_KV_NAMESPACE_ID": os.environ["PREVIEW_KV_NAMESPACE_ID"], + "https://congress-trade-preview..workers.dev": os.environ["PREVIEW_APP_BASE_URL"], + } + for old, new in replacements.items(): + text = text.replace(old, new) + path.write_text(text) + PY + env: + PREVIEW_D1_DATABASE_ID: ${{ secrets.PREVIEW_D1_DATABASE_ID }} + PREVIEW_KV_NAMESPACE_ID: ${{ secrets.PREVIEW_KV_NAMESPACE_ID }} + PREVIEW_APP_BASE_URL: ${{ vars.PREVIEW_APP_BASE_URL || 'https://congress-trade-preview.workers.dev' }} + + - name: Deploy to Cloudflare Workers (preview) + id: deploy + run: | + DEPLOY_URL=$(bash scripts/deploy-preview.sh 2>&1 | tee /dev/stderr | grep -o 'https://[^ ]*\.workers\.dev' | tail -1 || true) + echo "deploy_url=${DEPLOY_URL:-${PREVIEW_APP_BASE_URL}}" >> "$GITHUB_OUTPUT" + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + PREVIEW_APP_BASE_URL: ${{ vars.PREVIEW_APP_BASE_URL || 'https://congress-trade-preview.workers.dev' }} + + - name: Comment deploy URL + if: github.event_name == 'push' + run: | + URL="${{ steps.deploy.outputs.deploy_url }}" + PR_NUMBER="$(gh pr list --head staging --json number -q '.[0].number')" + if [ -n "$PR_NUMBER" ]; then + gh pr comment "$PR_NUMBER" --body "Preview deployed: ${URL}" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..72cec3a7f --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,74 @@ +name: Deploy + +on: + workflow_dispatch: + inputs: + confirm: + description: "Type deploy-production to deploy congress.trade" + required: true + type: string + +# Never run two deploys at once; let an in-flight deploy finish rather than +# cancel it mid-deploy. +concurrency: + group: deploy-production + cancel-in-progress: false + +jobs: + deploy: + if: github.event.inputs.confirm == 'deploy-production' + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: app/package-lock.json + + - name: Configure private GitHub dependencies + env: + SHARED_PACKAGE_DEPLOY_KEY: ${{ secrets.SHARED_PACKAGE_DEPLOY_KEY }} + GH_PAT: ${{ secrets.GH_PAT }} + run: | + if [ -n "$SHARED_PACKAGE_DEPLOY_KEY" ]; then + mkdir -p "$HOME/.ssh" + printf '%s\n' "$SHARED_PACKAGE_DEPLOY_KEY" > "$HOME/.ssh/congress_trading_shared" + chmod 600 "$HOME/.ssh/congress_trading_shared" + ssh-keyscan github.com >> "$HOME/.ssh/known_hosts" + git config --global core.sshCommand "ssh -i $HOME/.ssh/congress_trading_shared -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" + git config --global url."ssh://git@github.com/".insteadOf "https://github.com/" + git config --global url."ssh://git@github.com/".insteadOf "git@github.com:" + git ls-remote ssh://git@github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + elif [ -n "$GH_PAT" ]; then + git config --global credential.helper store + printf 'https://x-access-token:%s@github.com\n' "$GH_PAT" > "$HOME/.git-credentials" + git config --global url."https://github.com/".insteadOf "ssh://git@github.com/" + git config --global url."https://github.com/".insteadOf "git@github.com:" + git ls-remote https://github.com/jaywedgeworth22/congress-trading-shared.git HEAD >/dev/null + else + echo "::error::SHARED_PACKAGE_DEPLOY_KEY or GH_PAT with read access to jaywedgeworth22/congress-trading-shared is required for npm ci." + exit 1 + fi + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run typecheck + + - name: Test + run: npm test + + - name: Deploy to Cloudflare Workers + run: bash scripts/ship.sh + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + ADMIN_TOKEN: ${{ secrets.ADMIN_TOKEN }} diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 000000000..a158479cc --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,51 @@ +name: Security + +on: + pull_request: + push: + branches: + - main + schedule: + - cron: "41 10 * * 1" + +jobs: + gitleaks: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + steps: + - name: Refuse untrusted PR source + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then + echo "::error::Fork PRs cannot run security scans." + exit 1 + fi + actor="${{ github.actor }}" + if printf '%s' "$actor" | grep -Eq '\[bot\]$'; then + case "$actor" in + 'cursor[bot]'|'dependabot[bot]') + echo "Trusted same-repo bot: $actor" + ;; + *) + echo "::error::Untrusted bot PRs cannot run security scans (untrusted bot: $actor)." + exit 1 + ;; + esac + fi + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Clean stale gitleaks installer temp files + shell: bash + run: | + set -euo pipefail + tmp_root="${TMPDIR:-/tmp}" + tmp_root="${tmp_root%/}" + rm -f "$tmp_root/gitleaks.tmp" + rm -rf "$tmp_root/gitleaks-8.24.3" + - uses: gitleaks/gitleaks-action@e0c47f4f8be36e29cdc102c57e68cb5cbf0e8d1e + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/uptime-monitor.yml b/.github/workflows/uptime-monitor.yml new file mode 100644 index 000000000..b467a5ec5 --- /dev/null +++ b/.github/workflows/uptime-monitor.yml @@ -0,0 +1,91 @@ +name: Uptime Monitor + +on: + schedule: + - cron: '*/5 * * * *' # Every 5 minutes + workflow_dispatch: {} + +permissions: + contents: read + issues: write + +jobs: + ping: + runs-on: ubuntu-latest + steps: + - name: Health check + id: check + run: | + BODY="$(mktemp)" + HTTP_CODE="000" + if ! HTTP_CODE="$(curl -sS -o "$BODY" -w "%{http_code}" https://congress.trade/api/health)"; then + echo "status=fail" >> "$GITHUB_OUTPUT" + echo "http_code=000" >> "$GITHUB_OUTPUT" + echo "reason=curl failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "http_code=$HTTP_CODE" >> "$GITHUB_OUTPUT" + if [ "$HTTP_CODE" != "200" ]; then + echo "status=fail" >> "$GITHUB_OUTPUT" + echo "reason=unexpected HTTP status" >> "$GITHUB_OUTPUT" + elif ! REASON="$(node - "$BODY" <<'NODE' + const fs = require('node:fs'); + const body = fs.readFileSync(process.argv[2], 'utf8'); + let health; + try { + health = JSON.parse(body); + } catch { + console.log('health response is not valid JSON'); + process.exit(1); + } + if (health.ok !== true) { + console.log('health.ok is not true'); + process.exit(1); + } + if (health.db !== true) { + console.log('health.db is not true'); + process.exit(1); + } + console.log('ok'); + NODE + )"; then + echo "status=fail" >> "$GITHUB_OUTPUT" + echo "reason=$REASON" >> "$GITHUB_OUTPUT" + else + echo "status=ok" >> "$GITHUB_OUTPUT" + echo "reason=ok" >> "$GITHUB_OUTPUT" + fi + { + echo "health_body<> "$GITHUB_OUTPUT" + + - name: Open issue on failure + if: steps.check.outputs.status == 'fail' + run: | + title="Uptime Alert: congress.trade health returned HTTP ${{ steps.check.outputs.http_code }}" + body_file="$(mktemp)" + now="$(date -u)" + cat > "$body_file" < 503) # - STRIPE_WEBHOOK_SECRET: whsec_… from the webhook endpoint you create for diff --git a/app/docs/fmp-data-sharing.md b/app/docs/fmp-data-sharing.md index 91a45be80..82ddfbe2a 100644 --- a/app/docs/fmp-data-sharing.md +++ b/app/docs/fmp-data-sharing.md @@ -318,6 +318,20 @@ provider — see the cross-app data-sharing plan. ## Bulk snapshot export (full-history bootstrap / catch-up) +Before App B hardcodes a new route or export shape, it should read the +machine-readable integration manifest: + +``` +GET https://congress.trade/api/export/capabilities +Headers: Authorization: Bearer +``` + +The response lists the current cross-app contract version, supported import +payload slots, import limits, read endpoints, PIT score export settings, +bulk-snapshot table names, placebo exports, and whether the App B return path is +configured. It intentionally reports only boolean secret/config status; it never +echoes token values or peer URLs. + The per-ticker reads above are for incremental, one-symbol cache-aside. To **bootstrap from scratch** or **catch up after a downtime gap**, App B can pull a daily, date-partitioned NDJSON snapshot of the whole market-data set instead of @@ -339,7 +353,6 @@ schema, and a per-table `downloadPath`: { "generatedAt": "2026-06-25T04:01:00.000Z", "snapshotDate": "2026-06-25", - "snapshotDate": "2026-06-25", "runId": "9f3c…", // unique per run; pinned into downloadPath "format": "ndjson", "tables": { diff --git a/app/package-lock.json b/app/package-lock.json index fbad12165..b3bd2d2e9 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -8,6 +8,8 @@ "name": "congress-feed", "version": "0.1.0", "dependencies": { + "@jaywedgeworth22/congress-trading-shared": "git+https://github.com/jaywedgeworth22/congress-trading-shared.git#a33dfd3fea5bd0fa2f10f4aab5e32d4eff144f14", + "@sentry/cloudflare": "^10.62.0", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", @@ -191,7 +193,7 @@ "version": "4.20260625.1", "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260625.1.tgz", "integrity": "sha512-asH0RhPHiNu/IUSssyiOJYAcGqysy0DJpO9fihC6KATaayD9CE1E9bgNQozTLUraxrCT2qkM4CBOIcV0M5NPJw==", - "dev": true, + "devOptional": true, "license": "MIT OR Apache-2.0" }, "node_modules/@cspotcode/source-map-support": { @@ -1173,6 +1175,15 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@jaywedgeworth22/congress-trading-shared": { + "version": "1.0.0", + "resolved": "git+https://github.com/jaywedgeworth22/congress-trading-shared.git#a33dfd3fea5bd0fa2f10f4aab5e32d4eff144f14", + "integrity": "sha512-6y6eDQwk7QwgGkoJHGLFTXFVv2xpnBsydLi7nAfZ1quZ7W63Uh3bEauIu3mCWTtyOxVYu/1aAY7DtVaEaf3GCA==", + "license": "UNLICENSED", + "dependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1220,6 +1231,15 @@ "@emnapi/runtime": "^1.7.1" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@oxc-project/types": { "version": "0.133.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", @@ -1534,6 +1554,36 @@ "dev": true, "license": "MIT" }, + "node_modules/@sentry/cloudflare": { + "version": "10.62.0", + "resolved": "https://registry.npmjs.org/@sentry/cloudflare/-/cloudflare-10.62.0.tgz", + "integrity": "sha512-oHDpXXiO3XpBO2cHiTRQpSrtQOQrsU9JsO3TZ6ukdd24IUE6Tkc3l7hWdwzKqId3nTWP1Ef0Fr+offsrEGJ6UA==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@sentry/core": "10.62.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@cloudflare/workers-types": "^4.x" + }, + "peerDependenciesMeta": { + "@cloudflare/workers-types": { + "optional": true + } + } + }, + "node_modules/@sentry/core": { + "version": "10.62.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.62.0.tgz", + "integrity": "sha512-tV69fMg2sS5DUFmQSnS7Jd5qJAp0izxwcsvBVz2ieTM9VMRi99IfOSYW9UYr3p1yfuksk41kefN5PEbeedUE+A==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@sindresorhus/is": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", @@ -3088,7 +3138,6 @@ "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/app/package.json b/app/package.json index 4d4da8b80..49db729bf 100644 --- a/app/package.json +++ b/app/package.json @@ -17,6 +17,8 @@ "preview:deploy": "bash scripts/deploy-preview.sh" }, "dependencies": { + "@jaywedgeworth22/congress-trading-shared": "git+https://github.com/jaywedgeworth22/congress-trading-shared.git#a33dfd3fea5bd0fa2f10f4aab5e32d4eff144f14", + "@sentry/cloudflare": "^10.62.0", "fflate": "^0.8.3", "hono": "^4.6.0", "node-html-parser": "^7.1.0", diff --git a/app/src/admin/__tests__/diagnostics.test.ts b/app/src/admin/__tests__/diagnostics.test.ts index a6f1aa468..d0f59136b 100644 --- a/app/src/admin/__tests__/diagnostics.test.ts +++ b/app/src/admin/__tests__/diagnostics.test.ts @@ -26,6 +26,33 @@ function fakeDb() { ] as T[], }; } + if (/FROM securities_ref/i.test(sql) && /CASE\s+WHEN lower\(source\)/i.test(sql)) { + return { + results: [ + { + provider: 'massive', + calls_total: 2, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24T11:30:00.000Z', + errors_last_24h: 0, + }, + ] as T[], + }; + } + if (/FROM securities_ref/i.test(sql) && /COUNT\(\*\) AS calls_total/i.test(sql)) { + return { + results: [ + { + calls_total: 5, + calls_last_24h: 2, + calls_today: 1, + last_used_at: '2026-06-24T11:00:00.000Z', + errors_last_24h: 1, + }, + ] as T[], + }; + } if (/FROM securities_ref/i.test(sql) && /enrichment_error/i.test(sql)) { return { results: [ @@ -37,6 +64,42 @@ function fakeDb() { ] as T[], }; } + if (/FROM price_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 7, + calls_last_24h: 20, + calls_today: 3, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM spx_eod/i.test(sql)) { + return { + results: [ + { + calls_total: 100, + calls_last_24h: 1, + calls_today: 1, + last_used_at: '2026-06-24', + }, + ] as T[], + }; + } + if (/FROM tx_performance/i.test(sql)) { + return { + results: [ + { + calls_total: 50, + calls_last_24h: 10, + calls_today: 5, + last_used_at: '2026-06-24T12:30:00.000Z', + }, + ] as T[], + }; + } if (/FROM deliveries/i.test(sql)) return { results: [] as T[] }; if (/FROM review_queue/i.test(sql)) return { results: [] as T[] }; if (/FROM client_commands/i.test(sql)) return { results: [] as T[] }; @@ -72,6 +135,8 @@ describe('admin diagnostics API', () => { ADMIN_TOKEN: 'admin-secret', GEMINI_API_KEY: 'gemini-secret', FMP_API_KEY: 'fmp-secret', + MASSIVE_API_KEY: 'massive-secret', + PRICE_PROVIDER: 'massive', DB: fakeDb(), } as never, ); @@ -91,10 +156,15 @@ describe('admin diagnostics API', () => { callsToday: 1, }), expect.objectContaining({ id: 'source:house', status: 'ok', callsToday: 2 }), + expect.objectContaining({ id: 'provider:massive', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:prices', status: 'ok', configured: true, callsToday: 3 }), + expect.objectContaining({ id: 'cache:spx', status: 'ok', configured: true, callsToday: 1 }), + expect.objectContaining({ id: 'cache:performance', status: 'ok', configured: true, callsToday: 5 }), ]), ); expect(JSON.stringify(body)).not.toContain('gemini-secret'); expect(JSON.stringify(body)).not.toContain('fmp-secret'); + expect(JSON.stringify(body)).not.toContain('massive-secret'); expect(body.errors).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/app/src/admin/routes.ts b/app/src/admin/routes.ts index 40af92cdf..8f0a4b183 100644 --- a/app/src/admin/routes.ts +++ b/app/src/admin/routes.ts @@ -1278,63 +1278,6 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { note: runtimeSecrets.FMP_API_KEY ? 'Enrichment rows refreshed' : 'FMP_API_KEY is not available to this Worker runtime', }); - const priceRows = await optionalAll<{ - calls_total: number; - last_price_date: string | null; - }>( - c.env, - `SELECT COUNT(*) AS calls_total, MAX(date) AS last_price_date FROM price_eod`, - ); - const priceRow = priceRows[0]; - const marketProviders: Array<{ id: string; label: string; configured: boolean; note: string }> = [ - { - id: 'provider:massive', - label: 'Massive Market Data', - configured: !!runtimeSecrets.MASSIVE_API_KEY, - note: runtimeSecrets.MASSIVE_API_KEY ? 'Configured as price/enrichment fallback' : 'MASSIVE_API_KEY is not available to this Worker runtime', - }, - { - id: 'provider:intrinio', - label: 'Intrinio Enrichment', - configured: !!runtimeSecrets.INTRINIO_API_KEY, - note: runtimeSecrets.INTRINIO_API_KEY ? 'Configured as enrichment fallback' : 'INTRINIO_API_KEY is not available to this Worker runtime', - }, - { - id: 'provider:twelvedata', - label: 'TwelveData Enrichment', - configured: !!runtimeSecrets.TWELVEDATA_API_KEY, - note: runtimeSecrets.TWELVEDATA_API_KEY ? 'Configured as enrichment fallback' : 'TWELVEDATA_API_KEY is not available to this Worker runtime', - }, - { - id: 'provider:finnhub', - label: 'Finnhub Enrichment', - configured: !!runtimeSecrets.FINNHUB_API_KEY, - note: runtimeSecrets.FINNHUB_API_KEY ? 'Configured as enrichment fallback' : 'FINNHUB_API_KEY is not available to this Worker runtime', - }, - { - id: 'provider:logodev', - label: 'Logo.dev', - configured: !!runtimeSecrets.LOGODEV_PUBLISHABLE_KEY, - note: runtimeSecrets.LOGODEV_PUBLISHABLE_KEY ? 'Ticker logo proxy token available' : 'LOGODEV_PUBLISHABLE_KEY is not available to this Worker runtime', - }, - ]; - for (const provider of marketProviders) { - connections.push({ - id: provider.id, - label: provider.label, - status: provider.configured ? 'ok' : 'warn', - configured: provider.configured, - lastUsedAt: null, - callsTotal: provider.id === 'provider:massive' ? priceRow?.calls_total ?? 0 : 0, - callsLast24h: 0, - callsToday: 0, - errorsLast24h: 0, - note: provider.id === 'provider:massive' && priceRow?.last_price_date - ? `${provider.note}; latest cached price date ${priceRow.last_price_date}` - : provider.note, - }); - } - const appBReceivedRows = await optionalAll<{ imported_refs: number; fundamentals_rows: number; @@ -1387,6 +1330,155 @@ export function buildAdminRouter(): Hono<{ Bindings: Env }> { : 'APP_B_IMPORT_URL/APP_B_INGEST_TOKEN is missing or incomplete', }); + const providerRows = await optionalAll<{ + provider: string; + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + errors_last_24h: number; + }>( + c.env, + `SELECT CASE + WHEN lower(source) LIKE '%massive%' THEN 'massive' + WHEN lower(source) LIKE '%intrinio%' THEN 'intrinio' + WHEN lower(source) LIKE '%twelvedata%' THEN 'twelvedata' + WHEN lower(source) LIKE '%finnhub%' THEN 'finnhub' + WHEN lower(source) LIKE '%edgar%' THEN 'edgar' + ELSE 'other' + END AS provider, + COUNT(*) AS calls_total, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN enriched_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(enriched_at) AS last_used_at, + SUM(CASE WHEN enrichment_error IS NOT NULL AND enrichment_error != '' AND enriched_at >= ? THEN 1 ELSE 0 END) AS errors_last_24h + FROM securities_ref + WHERE source IS NOT NULL AND source != '' + GROUP BY provider`, + [last24, today, last24], + ); + const providerUsage = new Map(providerRows.map((row) => [row.provider, row])); + const addMarketProvider = (id: string, label: string, configured: boolean, note: string) => { + const row = providerUsage.get(id); + connections.push({ + id: `provider:${id}`, + label, + status: connectionStatus(configured, row?.errors_last_24h ?? 0, row?.last_used_at ?? null), + configured, + lastUsedAt: row?.last_used_at ?? null, + callsTotal: row?.calls_total ?? 0, + callsLast24h: row?.calls_last_24h ?? 0, + callsToday: row?.calls_today ?? 0, + errorsLast24h: row?.errors_last_24h ?? 0, + note, + }); + }; + addMarketProvider('massive', 'Massive Market Data', !!runtimeSecrets.MASSIVE_API_KEY, runtimeSecrets.MASSIVE_API_KEY ? 'Reference/price fallback configured' : 'MASSIVE_API_KEY is not available to this Worker runtime'); + addMarketProvider('intrinio', 'Intrinio Reference Data', !!runtimeSecrets.INTRINIO_API_KEY, runtimeSecrets.INTRINIO_API_KEY ? 'Reference fallback configured' : 'INTRINIO_API_KEY is not available to this Worker runtime'); + addMarketProvider('twelvedata', 'Twelve Data Reference', !!runtimeSecrets.TWELVEDATA_API_KEY, runtimeSecrets.TWELVEDATA_API_KEY ? 'Reference fallback configured' : 'TWELVEDATA_API_KEY is not available to this Worker runtime'); + addMarketProvider('finnhub', 'Finnhub Reference', !!runtimeSecrets.FINNHUB_API_KEY, runtimeSecrets.FINNHUB_API_KEY ? 'Reference fallback configured' : 'FINNHUB_API_KEY is not available to this Worker runtime'); + addMarketProvider('edgar', 'SEC EDGAR Reference', true, 'Free fallback; no secret required'); + + connections.push({ + id: 'provider:logodev', + label: 'Logo.dev', + status: runtimeSecrets.LOGODEV_PUBLISHABLE_KEY ? 'ok' : 'warn', + configured: !!runtimeSecrets.LOGODEV_PUBLISHABLE_KEY, + lastUsedAt: null, + callsTotal: 0, + callsLast24h: 0, + callsToday: 0, + errorsLast24h: 0, + note: runtimeSecrets.LOGODEV_PUBLISHABLE_KEY ? 'Ticker logo proxy token available' : 'LOGODEV_PUBLISHABLE_KEY is not available to this Worker runtime', + }); + + const priceRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(DISTINCT ticker) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM price_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const priceRow = priceRows[0]; + const hasPriceProvider = !!(runtimeSecrets.FMP_API_KEY || runtimeSecrets.MASSIVE_API_KEY); + connections.push({ + id: 'cache:prices', + label: 'Asset Price Cache', + status: connectionStatus(hasPriceProvider, 0, priceRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: priceRow?.last_used_at ?? null, + callsTotal: priceRow?.calls_total ?? 0, + callsLast24h: priceRow?.calls_last_24h ?? 0, + callsToday: priceRow?.calls_today ?? 0, + errorsLast24h: 0, + note: hasPriceProvider + ? `PRICE_PROVIDER=${c.env.PRICE_PROVIDER || 'fmp'}; counts show cached assets/rows, not raw API calls` + : 'No FMP_API_KEY or MASSIVE_API_KEY configured for price history', + }); + + const spxRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN date >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(date) AS last_used_at + FROM spx_eod`, + [last24.slice(0, 10), today.slice(0, 10)], + ); + const spxRow = spxRows[0]; + connections.push({ + id: 'cache:spx', + label: 'S&P Benchmark Cache', + status: connectionStatus(hasPriceProvider, 0, spxRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: spxRow?.last_used_at ?? null, + callsTotal: spxRow?.calls_total ?? 0, + callsLast24h: spxRow?.calls_last_24h ?? 0, + callsToday: spxRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'SPY-adjusted close history used as the S&P comparison baseline', + }); + + const perfRows = await optionalAll<{ + calls_total: number; + calls_last_24h: number; + calls_today: number; + last_used_at: string | null; + }>( + c.env, + `SELECT COUNT(*) AS calls_total, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_last_24h, + SUM(CASE WHEN computed_at >= ? THEN 1 ELSE 0 END) AS calls_today, + MAX(computed_at) AS last_used_at + FROM tx_performance`, + [last24, today], + ); + const perfRow = perfRows[0]; + connections.push({ + id: 'cache:performance', + label: 'Trade Performance Anchors', + status: connectionStatus(hasPriceProvider, 0, perfRow?.last_used_at ?? null), + configured: hasPriceProvider, + lastUsedAt: perfRow?.last_used_at ?? null, + callsTotal: perfRow?.calls_total ?? 0, + callsLast24h: perfRow?.calls_last_24h ?? 0, + callsToday: perfRow?.calls_today ?? 0, + errorsLast24h: 0, + note: 'Required for per-trade and member S&P-relative performance', + }); + const webhooks = await optionalAll<{ calls_total: number; calls_last_24h: number; diff --git a/app/src/delivery/__tests__/queueRetry.test.ts b/app/src/delivery/__tests__/queueRetry.test.ts index 34ca5b9e4..0d886be69 100644 --- a/app/src/delivery/__tests__/queueRetry.test.ts +++ b/app/src/delivery/__tests__/queueRetry.test.ts @@ -4,6 +4,14 @@ * webhook retry does not fan out to every subscriber again. */ import { describe, it, expect, vi, afterEach } from 'vitest'; + +// Sentry's queue instrumentation requires AsyncLocalStorage which isn't +// available in vitest. Mock withSentry as a pass-through so tests that call +// worker.queue() directly don't crash on isolation-scope setup. +vi.mock('@sentry/cloudflare', () => ({ + withSentry: (_opts: unknown, handler: unknown) => handler, +})); + import worker from '../../index'; import type { Env, QueueMessage } from '../../shared/types'; diff --git a/app/src/export/__tests__/routes.test.ts b/app/src/export/__tests__/routes.test.ts index 688b47be9..5836af995 100644 --- a/app/src/export/__tests__/routes.test.ts +++ b/app/src/export/__tests__/routes.test.ts @@ -8,7 +8,7 @@ * run-scoped object key). */ -import { describe, it, expect } from 'vitest'; +import { afterEach, describe, it, expect, vi } from 'vitest'; import { buildExportRouter } from '../routes'; import { manifestObjectKey, snapshotObjectKey } from '../snapshot'; @@ -84,6 +84,10 @@ function baseEnv(extra: Record = {}) { const TODAY = new Date().toISOString().slice(0, 10); +afterEach(() => { + vi.unstubAllGlobals(); +}); + function manifestSeed(date: string) { return { [manifestObjectKey(date)]: JSON.stringify({ @@ -115,6 +119,85 @@ describe('GET /api/export/bulk-snapshot — auth', () => { }); }); +describe('GET /api/export/capabilities', () => { + it('401 without the scoped ingest token', async () => { + expect((await req('/capabilities', baseEnv())).status).toBe(401); + }); + + it('returns the cross-app contract without leaking configured secret values', async () => { + const res = await req( + '/capabilities', + baseEnv({ + APP_B_IMPORT_URL: 'https://app-b.example/api/admin/securities/import', + APP_B_INGEST_TOKEN: 'peer-secret', + IMPORT_MAX_REFS: '123', + }), + TOKEN, + ); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).not.toContain(TOKEN); + expect(text).not.toContain('peer-secret'); + expect(text).not.toContain('https://app-b.example'); + const body = JSON.parse(text) as { + contractVersion: string; + configured: { ingestToken: boolean; appBReturnPath: boolean }; + endpoints: { imports: { securities: { limits: { refs: number }; accepts: string[] } }; exports: { pitScores: { scoreVersion: string } } }; + }; + expect(body.contractVersion).toBe('congress-trade-crossapp-v1'); + expect(body.configured).toEqual({ ingestToken: true, appBReturnPath: true }); + expect(body.endpoints.imports.securities.limits.refs).toBe(123); + expect(body.endpoints.imports.securities.accepts).toContain('fundamentals'); + expect(body.endpoints.exports.pitScores.scoreVersion).toBe('congress-pit-v2'); + }); + + it('reports configuration resolved from Infisical secrets', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (url: string) => { + if (url.endsWith('/api/v1/auth/universal-auth/login')) { + return Response.json({ accessToken: 'infisical-token' }); + } + if (url.includes('/api/v3/secrets/raw')) { + return Response.json({ + secrets: [ + { secretKey: 'INGEST_TOKEN', secretValue: TOKEN }, + { secretKey: 'APP_B_IMPORT_URL', secretValue: 'https://app-b.example/api/admin/securities/import' }, + { secretKey: 'APP_B_INGEST_TOKEN', secretValue: 'peer-secret' }, + ], + }); + } + return new Response('not found', { status: 404 }); + }), + ); + + const res = await req( + '/capabilities', + baseEnv({ + INGEST_TOKEN: undefined, + INFISICAL_BASE_URL: 'https://infisical.test', + INFISICAL_ENV: 'prod', + INFISICAL_APP_PROJECT_ID: 'export-capabilities-app', + INFISICAL_APP_CLIENT_ID: 'app-client', + INFISICAL_APP_CLIENT_SECRET: 'app-secret', + }), + TOKEN, + ); + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).not.toContain(TOKEN); + expect(text).not.toContain('peer-secret'); + expect(text).not.toContain('https://app-b.example'); + const body = JSON.parse(text) as { + configured: { ingestToken: boolean; appBReturnPath: boolean }; + peerSharing: { appBImportUrlConfigured: boolean; appBIngestTokenConfigured: boolean }; + }; + expect(body.configured).toEqual({ ingestToken: true, appBReturnPath: true }); + expect(body.peerSharing.appBImportUrlConfigured).toBe(true); + expect(body.peerSharing.appBIngestTokenConfigured).toBe(true); + }); +}); + describe('GET /api/export/congress-pit-scores', () => { it('401 without a token', async () => { expect((await req('/congress-pit-scores', baseEnv())).status).toBe(401); diff --git a/app/src/export/routes.ts b/app/src/export/routes.ts index 844c63e4a..34ca281ec 100644 --- a/app/src/export/routes.ts +++ b/app/src/export/routes.ts @@ -28,7 +28,7 @@ import { Hono } from 'hono'; import type { Env } from '../shared/types'; import { constantTimeEqual } from '../auth/tokens'; -import { resolveSecret } from '../secrets/infisical'; +import { resolveSecret, resolveSecrets } from '../secrets/infisical'; import { runBulkSnapshot, readManifest, @@ -41,14 +41,37 @@ import { buildPitScoreExport, parsePitScoreQuery, pitScoreRowsToNdjson, + PIT_PLACEBOS, + PIT_SCORE_VERSION, } from './pitScores'; -/** Env augmented with the scoped cross-app ingest token (mirrors admin/routes). */ -type ExportEnv = Env & { INGEST_TOKEN?: string }; +/** Env augmented with cross-app sharing config (mirrors admin/share routes). */ +type ExportEnv = Env & { INGEST_TOKEN?: string; APP_B_IMPORT_URL?: string; APP_B_INGEST_TOKEN?: string }; const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; const RUN_ID_RE = /^[A-Za-z0-9-]{1,64}$/; // crypto.randomUUID() shape (hex + dashes) const VALID_TABLES = new Set(SNAPSHOT_TABLES.map((t) => t.name)); +const CAPABILITIES_VERSION = 'congress-trade-crossapp-v1'; + +const IMPORT_DEFAULT_LIMITS = { + bytes: 1_500_000, + refs: 2_000, + spx: 5_000, + prices: 100, + closesPerTicker: 1_500, + insider: 5_000, + shortVolume: 5_000, +}; + +const IMPORT_MAX_LIMITS = { + bytes: 3_000_000, + refs: 5_000, + spx: 10_000, + prices: 250, + closesPerTicker: 3_000, + insider: 10_000, + shortVolume: 10_000, +}; function todayUtc(now = new Date()): string { return now.toISOString().slice(0, 10); @@ -94,9 +117,134 @@ function shapeManifest(manifest: SnapshotManifest, tables: SnapshotTableName[]): }; } +function positiveIntSetting(raw: string | undefined, fallback: number, max: number): number { + const n = Number.parseInt(raw ?? '', 10); + if (!Number.isFinite(n) || n <= 0) return fallback; + return Math.min(Math.floor(n), max); +} + +function integrationImportLimits(env: ExportEnv): typeof IMPORT_DEFAULT_LIMITS { + return { + bytes: positiveIntSetting(env.IMPORT_MAX_BYTES, IMPORT_DEFAULT_LIMITS.bytes, IMPORT_MAX_LIMITS.bytes), + refs: positiveIntSetting(env.IMPORT_MAX_REFS, IMPORT_DEFAULT_LIMITS.refs, IMPORT_MAX_LIMITS.refs), + spx: positiveIntSetting(env.IMPORT_MAX_SPX, IMPORT_DEFAULT_LIMITS.spx, IMPORT_MAX_LIMITS.spx), + prices: positiveIntSetting(env.IMPORT_MAX_PRICES, IMPORT_DEFAULT_LIMITS.prices, IMPORT_MAX_LIMITS.prices), + closesPerTicker: positiveIntSetting( + env.IMPORT_MAX_CLOSES_PER_TICKER, + IMPORT_DEFAULT_LIMITS.closesPerTicker, + IMPORT_MAX_LIMITS.closesPerTicker, + ), + insider: positiveIntSetting(env.IMPORT_MAX_INSIDER, IMPORT_DEFAULT_LIMITS.insider, IMPORT_MAX_LIMITS.insider), + shortVolume: positiveIntSetting(env.IMPORT_MAX_SHORT_VOLUME, IMPORT_DEFAULT_LIMITS.shortVolume, IMPORT_MAX_LIMITS.shortVolume), + }; +} + +async function integrationCapabilities(env: ExportEnv): Promise> { + const runtimeSecrets = await resolveSecrets(env, ['INGEST_TOKEN', 'APP_B_IMPORT_URL', 'APP_B_INGEST_TOKEN']); + const configured = { + ingestToken: Boolean(runtimeSecrets.INGEST_TOKEN), + appBReturnPath: Boolean(runtimeSecrets.APP_B_IMPORT_URL && runtimeSecrets.APP_B_INGEST_TOKEN), + }; + return { + app: 'congress.trade', + generatedAt: new Date().toISOString(), + contractVersion: CAPABILITIES_VERSION, + auth: { + scheme: 'bearer', + tokenName: 'INGEST_TOKEN', + requiredFor: [ + '/api/admin/securities/import', + '/api/export/capabilities', + '/api/export/congress-pit-scores', + '/api/export/bulk-snapshot', + '/api/export/bulk-snapshot/file', + ], + }, + configured, + peerSharing: { + role: 'source-of-truth-for-congressional-disclosures-and-point-in-time-scores', + appBReturnPathConfigured: configured.appBReturnPath, + appBImportUrlConfigured: Boolean(runtimeSecrets.APP_B_IMPORT_URL), + appBIngestTokenConfigured: Boolean(runtimeSecrets.APP_B_INGEST_TOKEN), + noEchoPolicy: 'Only freshly fetched local deltas are pushed to App B; App B-origin imports are not echoed back.', + }, + endpoints: { + imports: { + securities: { + method: 'POST', + path: '/api/admin/securities/import', + auth: 'bearer INGEST_TOKEN', + accepts: ['refs', 'prices', 'spx', 'insider', 'shortVolume', 'fundamentals', 'analyst', 'origin'], + limits: integrationImportLimits(env), + }, + }, + publicReads: { + marketBundle: { method: 'GET', path: '/api/market/bundle/:ticker?from=&to=' }, + marketRef: { method: 'GET', path: '/api/market/ref/:ticker' }, + marketRefs: { method: 'GET', path: '/api/market/refs?tickers=AAPL,MSFT' }, + prices: { method: 'GET', path: '/api/market/prices/:ticker?from=&to=' }, + spx: { method: 'GET', path: '/api/market/spx?from=&to=' }, + insider: { method: 'GET', path: '/api/market/insider/:ticker?from=&to=' }, + shortVolume: { method: 'GET', path: '/api/market/short-volume/:ticker?from=&to=' }, + fundamentals: { method: 'GET', path: '/api/market/fundamentals/:ticker?from=&to=' }, + analyst: { method: 'GET', path: '/api/market/analyst/:ticker?from=&to=' }, + transactions: { method: 'GET', path: '/api/transactions?cursor=&limit=&member=&ticker=&type=&chamber=' }, + }, + analytics: { + tickerLeaderboard: { method: 'GET', path: '/api/analytics/ticker-leaderboard?window=&rankBy=' }, + clusterBuys: { method: 'GET', path: '/api/analytics/cluster-buys?window=' }, + memberLeaderboard: { method: 'GET', path: '/api/analytics/member-leaderboard?window=&rankBy=' }, + memberPerformance: { method: 'GET', path: '/api/analytics/member/:filerId/performance?from=&to=' }, + conviction: { method: 'GET', path: '/api/analytics/conviction?ticker=&window=' }, + tickerBacktest: { method: 'GET', path: '/api/analytics/ticker/:ticker/backtest?from=&to=' }, + conflicts: { method: 'GET', path: '/api/analytics/conflicts?ticker=§or=' }, + }, + exports: { + pitScores: { + method: 'GET', + path: '/api/export/congress-pit-scores?from=&to=&ticker=&cursor=&limit=&format=json|ndjson&placebo=&source=&minConf=', + auth: 'bearer INGEST_TOKEN', + scoreVersion: PIT_SCORE_VERSION, + maxLimit: 500, + placebosAvailable: PIT_PLACEBOS, + }, + bulkSnapshot: { + method: 'GET', + path: '/api/export/bulk-snapshot?date=&tables=&format=ndjson', + auth: 'bearer INGEST_TOKEN', + format: 'ndjson', + tables: SNAPSHOT_TABLES.map((t) => ({ name: t.name, keyColumns: t.keyCols })), + }, + bulkSnapshotFile: { + method: 'GET', + path: '/api/export/bulk-snapshot/file?date=&runId=&table=', + auth: 'bearer INGEST_TOKEN', + format: 'ndjson', + }, + }, + }, + recommendedSync: { + bootstrap: 'Pull /api/export/bulk-snapshot, persist manifest runId/objectKeys, then stream each downloadPath.', + incrementalMarketData: 'Use /api/market/* reads as a cache-aside tier before paid providers.', + congressionalSignals: 'Use /api/export/congress-pit-scores for historical validation and /api/analytics/* for live overlays.', + writeBack: 'POST newly fetched refs/prices/spx/enrichment deltas to /api/admin/securities/import with origin set by the sender.', + }, + }; +} + export function buildExportRouter(): Hono<{ Bindings: ExportEnv }> { const r = new Hono<{ Bindings: ExportEnv }>(); + // --- GET /capabilities -------------------------------------------------- + // Token-gated machine-readable integration contract for sibling apps. App B + // can use this before hardcoding a new route, limit, or export shape. + r.get('/capabilities', async (c) => { + if (!(await isAuthorized(c.env, c.req.header('authorization')))) { + return c.json({ error: 'unauthorized' }, 401); + } + return c.json(await integrationCapabilities(c.env)); + }); + // --- GET /congress-pit-scores ------------------------------------------ // Token-gated point-in-time score export for App B historical validation. // Emits one row per (ticker, disclosure availability timestamp) observation. diff --git a/app/src/index.ts b/app/src/index.ts index eb2e32eb2..891863b0b 100644 --- a/app/src/index.ts +++ b/app/src/index.ts @@ -16,6 +16,7 @@ */ import { Hono } from 'hono'; +import * as Sentry from '@sentry/cloudflare'; import type { Env, QueueMessage } from './shared/types'; // Stage handlers owned by their feature modules. @@ -140,45 +141,54 @@ async function handleDeliveryMessage(env: Env, msg: QueueMessage): Promise } } -export default { - /** HTTP entrypoint. */ - fetch(request: Request, env: Env, ctx: ExecutionContext): Promise | Response { - return app.fetch(request, env, ctx); - }, +export default Sentry.withSentry( + (env: Env) => ({ + dsn: env.SENTRY_DSN, + // Send traces for a sample of transactions (0 = off, 1.0 = all). + // Defaults to 0; set to e.g. 0.1 for 10% sampling in production. + tracesSampleRate: 0, + }), + { + /** HTTP entrypoint. */ + fetch(request: Request, env: Env, ctx: ExecutionContext): Promise | Response { + return app.fetch(request, env, ctx); + }, - /** Cron entrypoint — runs every minute; watcher self-gates via shouldPollNow. - * Daily enrichment + price refresh self-gate via a KV date stamp. */ - async scheduled(_event: ScheduledController, env: Env, ctx: ExecutionContext): Promise { - await runWatcher(env, new Date()); - ctx.waitUntil(refreshSecrets(env).catch((err) => console.warn('infisical secret refresh failed:', (err as Error).message))); - ctx.waitUntil(maybeRunDailyJobs(env)); - // Autonomous cross-vendor agreement → auto-publish for a few newly-reviewed - // docs each minute (self-gates on AGREEMENT_AUTOPUBLISH_ENABLED; cron-safe). - ctx.waitUntil( - maybeRunAgreementAutopublish(env).catch((err) => - console.warn('agreement autopublish failed:', (err as Error).message), - ), - ); - }, + /** Cron entrypoint — runs every minute; watcher self-gates via shouldPollNow. + * Daily enrichment + price refresh self-gate via a KV date stamp. */ + async scheduled(_event: ScheduledController, env: Env, ctx: ExecutionContext): Promise { + await runWatcher(env, new Date()); + ctx.waitUntil(refreshSecrets(env).catch((err) => console.warn('infisical secret refresh failed:', (err as Error).message))); + ctx.waitUntil(maybeRunDailyJobs(env)); + // Autonomous cross-vendor agreement → auto-publish for a few newly-reviewed + // docs each minute (self-gates on AGREEMENT_AUTOPUBLISH_ENABLED; cron-safe). + ctx.waitUntil( + maybeRunAgreementAutopublish(env).catch((err) => + console.warn('agreement autopublish failed:', (err as Error).message), + ), + ); + }, - /** - * Queue consumer. Routes by the bound queue name to the ingest/delivery - * handlers. Messages are ack'd individually; failures retry per wrangler.toml. - */ - async queue(batch: MessageBatch, env: Env, _ctx: ExecutionContext): Promise { - const isDelivery = batch.queue.includes('delivery'); - for (const message of batch.messages) { - try { - if (isDelivery) { - await handleDeliveryMessage(env, message.body); - } else { - await handleIngestMessage(env, message.body); + /** + * Queue consumer. Routes by the bound queue name to the ingest/delivery + * handlers. Messages are ack'd individually; failures retry per wrangler.toml. + */ + async queue(batch, env: Env, _ctx: ExecutionContext): Promise { + const isDelivery = batch.queue.includes('delivery'); + for (const message of batch.messages) { + try { + const msg = message.body as QueueMessage; + if (isDelivery) { + await handleDeliveryMessage(env, msg); + } else { + await handleIngestMessage(env, msg); + } + message.ack(); + } catch (err) { + console.error(`queue ${batch.queue} message failed:`, (err as Error).message); + message.retry(); } - message.ack(); - } catch (err) { - console.error(`queue ${batch.queue} message failed:`, (err as Error).message); - message.retry(); } - } + }, }, -}; +); diff --git a/app/src/shared/types.ts b/app/src/shared/types.ts index b2e893f48..199cf1523 100644 --- a/app/src/shared/types.ts +++ b/app/src/shared/types.ts @@ -461,6 +461,8 @@ export interface Env { PRICE_PROVIDER?: string; /** HMAC key for signing outbound webhook payloads. */ WEBHOOK_SIGNING_KEY?: string; + /** Sentry DSN for error monitoring (Cloudflare Workers SDK). */ + SENTRY_DSN?: string; // --- End-user auth (public-site sign-in) --- /** Google OAuth client credentials for "Sign in with Google". */ diff --git a/app/src/ui/__tests__/dashboardHtml.test.ts b/app/src/ui/__tests__/dashboardHtml.test.ts index afe798834..652504eb3 100644 --- a/app/src/ui/__tests__/dashboardHtml.test.ts +++ b/app/src/ui/__tests__/dashboardHtml.test.ts @@ -77,6 +77,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('id="colChooser"'); expect(DASHBOARD_HTML).toContain('id="colChooserBody"'); expect(DASHBOARD_HTML).toContain('function resetCols('); + expect(DASHBOARD_HTML).toContain("var COL_ORDER_KEY = 'feed-cols-order-v1'"); + expect(DASHBOARD_HTML).toContain('function moveColumn('); + expect(DASHBOARD_HTML).toContain('Drag columns here to reorder the Trades table.'); + expect(DASHBOARD_HTML).toContain('draggable="true" data-colid'); // the new date/lag columns the user asked for expect(DASHBOARD_HTML).toContain("id: 'traded'"); expect(DASHBOARD_HTML).toContain("id: 'lag'"); @@ -99,6 +103,9 @@ describe('DASHBOARD_HTML', () => { it('keeps account sign-out discoverable from the account menu', () => { expect(DASHBOARD_HTML).toContain('id="acctMenuBtn"'); expect(DASHBOARD_HTML).toContain('Account'); + expect(DASHBOARD_HTML).toContain('themeMenuLabel'); + expect(DASHBOARD_HTML).not.toContain('id="themeToggle"'); + expect(DASHBOARD_HTML).toContain('white-space:nowrap; overflow:hidden; text-overflow:ellipsis;'); expect(DASHBOARD_HTML).toContain('Sign Out'); expect(DASHBOARD_HTML).toContain('function logout()'); }); @@ -186,8 +193,10 @@ describe('DASHBOARD_HTML', () => { it('uses published timing, tighter asset defaults, and source links in drawers', () => { expect(DASHBOARD_HTML).toContain("var sortKey = 'published'"); expect(DASHBOARD_HTML).toContain("var COL_HIDDEN_KEY = 'feed-cols-hidden-v2'"); - expect(DASHBOARD_HTML).toContain("var COL_WIDTH_KEY = 'feed-col-widths-v7'"); - expect(DASHBOARD_HTML).toContain("asset: estimatedColWidth('asset', 54, 48, 62)"); + expect(DASHBOARD_HTML).toContain("var COL_WIDTH_KEY = 'feed-col-widths-v8'"); + expect(DASHBOARD_HTML).toContain("asset: estimatedColWidth('asset', 48, 40, 54)"); + expect(DASHBOARD_HTML).not.toContain('width: max-content'); + expect(DASHBOARD_HTML).not.toContain('feed-col-widths-v7'); expect(DASHBOARD_HTML).toContain('function dateTimeCellHtml('); expect(DASHBOARD_HTML).toContain('date-time-cell'); expect(DASHBOARD_HTML).toContain('#feedTable.resizable th { text-align: center;'); @@ -210,9 +219,15 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('border-right: 1px solid color-mix'); expect(DASHBOARD_HTML).toContain('#feedTable .c-member'); expect(DASHBOARD_HTML).toContain('#feedTable .c-asset'); + expect(DASHBOARD_HTML).toContain(''); + expect(DASHBOARD_HTML).toContain('function syncFeedTableWidth('); + expect(DASHBOARD_HTML).toContain('.clip-text { display:block;'); expect(DASHBOARD_HTML).toContain('drawer-company-title'); expect(DASHBOARD_HTML).toContain('drawer-stack-grid'); expect(DASHBOARD_HTML).toContain('trend-members-grid'); + expect(DASHBOARD_HTML).toContain('.trend-grid2 { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr));'); + expect(DASHBOARD_HTML).toContain('.trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.6fr) minmax(0, .85fr);'); + expect(DASHBOARD_HTML).not.toContain('minmax(260px, .72fr)'); expect(DASHBOARD_HTML).toContain('buySellText('); expect(DASHBOARD_HTML).toContain('0% means matched the S&P'); expect(DASHBOARD_HTML).toContain('Unparsed Historical Filing'); @@ -234,8 +249,10 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('function useModelRows('); expect(DASHBOARD_HTML).toContain('function openReviewEditor('); expect(DASHBOARD_HTML).toContain('Review / Confirm'); - expect(DASHBOARD_HTML).toContain('Decision History'); - expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=100'"); + expect(DASHBOARD_HTML).toContain('Resolved Reviews'); + expect(DASHBOARD_HTML).toContain('All Filing Decisions'); + expect(DASHBOARD_HTML).toContain("fetch('/api/admin/ingestion-decisions?limit=200'"); + expect(DASHBOARD_HTML).toContain('function hasAdminToken()'); expect(DASHBOARD_HTML).toContain('function renderDecisionHistory('); expect(DASHBOARD_HTML).toContain('var DECISIONS'); expect(DASHBOARD_HTML).toContain('Use This Model'); @@ -329,6 +346,8 @@ describe('DASHBOARD_HTML', () => { expect(DASHBOARD_HTML).toContain('class="trend-grid2 timeliness-grid"'); expect(DASHBOARD_HTML).toContain('id="trLagDist" class="lag-dist"'); expect(DASHBOARD_HTML).toContain('class="late-filers-wrap"'); + expect(DASHBOARD_HTML).toContain('.timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(0, .92fr);'); + expect(DASHBOARD_HTML).not.toContain('minmax(280px, .92fr)'); expect(DASHBOARD_HTML).toContain('.late-filers-wrap { max-height: 232px; overflow: auto;'); expect(DASHBOARD_HTML).toContain('Disclosure lag is days between the transaction date and the official filing date.'); expect(DASHBOARD_HTML).toContain('Avg: mean number of days between transaction date and official filing date.'); diff --git a/app/src/ui/dashboardHtml.ts b/app/src/ui/dashboardHtml.ts index 5275b95f2..702af16fc 100644 --- a/app/src/ui/dashboardHtml.ts +++ b/app/src/ui/dashboardHtml.ts @@ -70,14 +70,12 @@ export const DASHBOARD_HTML = /* html */ ` } html[data-theme="light"] header.top { background: rgba(255,255,255,.72); } /* ---- theme toggle ---- */ - .theme-toggle { background: transparent; border: 1px solid var(--border); color: var(--text-dim); border-radius: 8px; padding: 6px 10px; cursor: pointer; font-size: 13px; line-height: 1; } - .theme-toggle:hover { color: var(--text); background: var(--panel); } /* ---- resizable feed columns ---- */ .table-wrap { overflow-x: auto; max-height: min(78vh, 920px); } - #feedTable.resizable { table-layout: fixed; width: max-content; min-width: 100%; } + #feedTable.resizable { table-layout: fixed; min-width: 100%; } #feedTable.resizable th, #feedTable.resizable td { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } #feedTable.resizable th { text-align: center; padding-right: 18px; } - #feedTable.resizable td > * { max-width: 100%; } + #feedTable.resizable td > * { max-width: 100%; min-width: 0; } #feedTable.resizable .asset-cell, #feedTable.resizable .member-cell { overflow: hidden; max-width: 100%; } #feedTable.resizable .asset-cell > div, @@ -176,6 +174,7 @@ export const DASHBOARD_HTML = /* html */ ` /* let the text shrink inside the (resizable, fixed-layout) cell and clip with an ellipsis instead of wrapping or hard-clipping mid-word */ .asset-cell > div { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .clip-text { display:block; min-width:0; max-width:100%; overflow:hidden; text-overflow:ellipsis; white-space:nowrap; } .tkr-logo { flex: 0 0 auto; display: inline-flex; align-items: center; justify-content: center; width: 22px; height: 22px; overflow: hidden; } .tkr-logo img { width: 100%; height: 100%; object-fit: contain; display: block; } /* "tile" = frosted-glass box; "transparent" = bare logo on the row surface. */ @@ -330,9 +329,10 @@ export const DASHBOARD_HTML = /* html */ ` .note { font-size:12px; color: var(--text-dim); margin-top:8px; line-height:1.5; } code { font-family: var(--mono); background: var(--bg); padding:1px 6px; border-radius:5px; font-size:12px; color: var(--accent); } /* ================= TRENDS / ANALYTICS ================= */ - .trend-grid2 { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; } + .trend-grid2 { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 18px; } + .trend-grid2 > *, .trend-members-grid > *, .trend-side-stack > *, .timeliness-grid > * { min-width: 0; } @media (max-width: 760px) { .trend-grid2 { grid-template-columns: 1fr; } } - .trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.75fr) minmax(260px, .72fr); gap:18px; align-items:start; } + .trend-members-grid { display:grid; grid-template-columns:minmax(0, 1.6fr) minmax(0, .85fr); gap:18px; align-items:start; } .trend-side-stack { display:grid; grid-template-columns:1fr; gap:18px; } @media (max-width: 920px) { .trend-members-grid { grid-template-columns:1fr; } } /* Roomier side drawer on tablets (mobile bottom-sheet still kicks in at 600px). */ @@ -361,7 +361,7 @@ export const DASHBOARD_HTML = /* html */ ` .hfill.buy { background: var(--buy); } .hfill.warn { background: var(--warn); } .hfill.sell { background: var(--sell); } .hbar .hval { width:120px; text-align:right; font-family: var(--mono); font-size:12px; color: var(--text-dim); } .hbar .hval .est-money { font-family: var(--mono); } - .timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(280px, .92fr); align-items: stretch; } + .timeliness-grid { margin-top: 8px; grid-template-columns: minmax(0, 1fr) minmax(0, .92fr); align-items: stretch; } .timeliness-panel { min-width: 0; } .timeliness-panel h3 { font-size: 13px; letter-spacing: 0; cursor: help; } .lag-dist { min-height: 232px; display: flex; flex-direction: column; justify-content: space-between; gap: 9px; } @@ -492,11 +492,15 @@ export const DASHBOARD_HTML = /* html */ ` .mini-date { display:flex; flex-direction:column; gap:2px; line-height:1.25; } .mini-date .subline { color:var(--text-dim); font-size:11px; } .mini-source-link { display:block; margin-top:2px; font-size:11px; font-weight:600; } - .colopts { display:flex; flex-wrap:wrap; gap:6px 4px; flex:1; } - .colopt { font-size:13px; color:var(--text); display:inline-flex; align-items:center; gap:5px; margin-right:12px; white-space:nowrap; cursor:pointer; } + .colopts { display:grid; grid-template-columns:repeat(auto-fit,minmax(170px,1fr)); gap:6px; flex:1; } + .colopt { font-size:13px; color:var(--text); display:inline-flex; align-items:center; gap:7px; margin-right:0; white-space:nowrap; cursor:pointer; min-width:0; } button.colopt { font-family:var(--sans); border:1px dashed var(--border); background:color-mix(in srgb,var(--panel-2) 65%,transparent); border-radius:999px; padding:3px 8px; } .colopt.locked { color:var(--text-dim); } .colopt.locked:hover { color:var(--text); border-color:color-mix(in srgb,var(--accent) 55%,var(--border)); } + .colopt.dragging { opacity:.45; border-color:var(--accent); } + .col-drag { color:var(--text-dim); cursor:grab; font-size:14px; line-height:1; } + .colopt input { flex:0 0 auto; } + .colopt-name { overflow:hidden; text-overflow:ellipsis; } .premium-mark { display:inline-flex; align-items:center; justify-content:center; border:1px solid color-mix(in srgb,var(--accent) 42%,var(--border)); background:color-mix(in srgb,var(--accent) 9%,transparent); color:var(--accent); border-radius:999px; padding:1px 6px; font-size:10px; font-weight:800; line-height:1.4; } .panel-note { flex-basis:100%; width:100%; color:var(--text-dim); font-size:12px; line-height:1.45; margin-bottom:4px; } .premium-count-note { margin-left:8px; color:var(--text-dim); } @@ -533,11 +537,11 @@ export const DASHBOARD_HTML = /* html */ ` .acct-menu-btn:hover { background:var(--panel-2); } .acct-menu-btn .acct-caret { color:var(--text-dim); font-size:11px; } .menu { position:relative; } - .menu-pop { position:absolute; right:0; top:38px; background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:6px; min-width:190px; box-shadow:0 12px 32px rgba(0,0,0,.38); display:none; z-index:30; } + .menu-pop { position:absolute; right:0; top:38px; background:var(--panel); border:1px solid var(--border); border-radius:10px; padding:6px; min-width:260px; max-width:min(320px, calc(100vw - 24px)); box-shadow:0 12px 32px rgba(0,0,0,.38); display:none; z-index:30; } .menu-pop.open { display:block; } .menu-pop button { display:block; width:100%; text-align:left; background:transparent; border:none; color:var(--text); padding:8px 10px; border-radius:7px; cursor:pointer; font-size:13px; font-family:var(--sans); } .menu-pop button:hover { background:var(--panel-2); } - .menu-pop .who { padding:6px 10px 8px; font-size:12px; color:var(--text-dim); border-bottom:1px solid var(--border); margin-bottom:5px; word-break:break-all; } + .menu-pop .who { padding:6px 10px 8px; font-size:12px; color:var(--text-dim); border-bottom:1px solid var(--border); margin-bottom:5px; white-space:nowrap; overflow:hidden; text-overflow:ellipsis; } .overlay { position:fixed; inset:0; background:rgba(4,8,16,.62); backdrop-filter:blur(3px); display:none; align-items:center; justify-content:center; z-index:50; padding:18px; } .overlay.open { display:flex; } .modal { background:var(--panel); border:1px solid var(--border); border-radius:16px; padding:26px; width:100%; max-width:430px; box-shadow:0 24px 60px rgba(0,0,0,.45); } @@ -574,7 +578,6 @@ export const DASHBOARD_HTML = /* html */ ` .brand { font-size: 15px; } #srcPill { display: none; } .pill { padding: 3px 7px; } - .theme-toggle { display:none; } nav.tabs { position: fixed; left: 0; right: 0; bottom: 0; margin: 0; width: 100%; max-width: 100%; @@ -1130,10 +1133,9 @@ export const DASHBOARD_HTML = /* html */ ` - +
-
@@ -1180,6 +1182,7 @@ export const DASHBOARD_HTML = /* html */ `
+
@@ -1345,10 +1348,10 @@ export const DASHBOARD_HTML = /* html */ `

Document Review & Model Comparison

-

Scanned / handwritten filings below the confidence threshold are held here until a human acts. Switch to Reviewed to see what was published / rejected / modified, and expand Models on any row to compare each model's confidence and reading.

+

Scanned / handwritten filings below the confidence threshold are held here until a human acts. Switch to Resolved Reviews to see what was published / rejected / modified. The All Filing Decisions table below includes auto-published filings too.

- +
@@ -1356,7 +1359,8 @@ export const DASHBOARD_HTML = /* html */ `
FiledDocStatusReasonPayload

Confirm promotes the read to the live feed; Manual lets you hand-key the rows (recorded as source=manual) when the automated read is wrong or too low-confidence; Reject discards it. Models / readings come from extraction_runs (populated by POST /api/admin/bakeoff). POST /api/admin/review/:docId {decision}

-

Decision History

+

All Filing Decisions

+

Append-only filing decisions, including clean auto-published filings that never entered the review queue.

@@ -1844,6 +1848,11 @@ function esc(s) { }); } function el(id) { return document.getElementById(id); } +function clipTextHtml(value, fallback, title) { + var text = String(value == null || value === '' ? (fallback || '—') : value); + var cls = text === '—' ? 'clip-text muted' : 'clip-text'; + return '' + esc(text) + ''; +} /* Strip stray HTML/entities some upstream datasets embed in asset descriptions (e.g. "
Rate/Coupon: 3.875%
…
"). */ @@ -1886,7 +1895,7 @@ function fmtMs(ms) { function applyTheme(t) { if (t === 'light') document.documentElement.setAttribute('data-theme', 'light'); else document.documentElement.removeAttribute('data-theme'); - var btn = el('themeToggle'); if (btn) btn.textContent = (t === 'light') ? '☀️' : '🌙'; + var label = el('themeMenuLabel'); if (label) label.textContent = (t === 'light') ? 'Light Mode' : 'Dark Mode'; } function toggleTheme() { var cur = document.documentElement.getAttribute('data-theme') === 'light' ? 'light' : 'dark'; @@ -2116,38 +2125,86 @@ var FEED_COLS = [ { id: 'traded', label: 'Traded', sort: 'txdate', def: true, cls: 'muted', tip: 'Date the trade was executed.', cell: function (r) { return dateCellHtml(r.txdate); } }, { id: 'lag', label: 'Lag', sort: 'lag', def: true, tip: 'Days between the trade and the filing (STOCK Act limit: 45).', cell: lagCellHtml }, { id: 'amount', label: 'Amount', sort: 'min', def: true, tip: 'STOCK Act bracket - an estimate, not an exact figure.', cell: amountCellHtml }, - { id: 'sector', label: 'Sector', sort: 'refSector', def: false, cls: 'muted', tier: 'premium', tip: 'Cross-referenced sector (FMP / SEC EDGAR). Blank until the asset is enriched.', cell: function (r) { return r.refSector ? esc(r.refSector) : '—'; } }, - { id: 'marketcap', label: 'Market Cap', sort: 'refMarketCap', def: false, tier: 'premium', tip: 'Market-cap size tier from enriched reference data.', cell: function (r) { return r.refMarketCapBucket ? esc(ownerLabel(r.refMarketCapBucket)) : '—'; } }, - { id: 'country', label: 'Country', sort: 'refCountry', def: false, cls: 'muted', tier: 'premium', tip: 'Country of issue from enriched reference data.', cell: function (r) { return r.refCountry ? esc(r.refCountry) : '—'; } }, - { id: 'owner', label: 'Owner', sort: 'owner', def: false, cls: 'muted', tip: 'Beneficial owner code reported on the filing.', cell: function (r) { return esc(ownerLabel(r.owner) || '—'); } }, + { id: 'sector', label: 'Sector', sort: 'refSector', def: false, cls: 'muted', tier: 'premium', tip: 'Cross-referenced sector (FMP / SEC EDGAR). Blank until the asset is enriched.', cell: function (r) { return clipTextHtml(r.refSector); } }, + { id: 'marketcap', label: 'Market Cap', sort: 'refMarketCap', def: false, tier: 'premium', tip: 'Market-cap size tier from enriched reference data.', cell: function (r) { return clipTextHtml(ownerLabel(r.refMarketCapBucket)); } }, + { id: 'country', label: 'Country', sort: 'refCountry', def: false, cls: 'muted', tier: 'premium', tip: 'Country of issue from enriched reference data.', cell: function (r) { return clipTextHtml(r.refCountry); } }, + { id: 'owner', label: 'Owner', sort: 'owner', def: false, cls: 'muted', tip: 'Beneficial owner code reported on the filing.', cell: function (r) { return clipTextHtml(ownerLabel(r.owner)); } }, { id: 'filed', label: 'Official Filed', sort: 'filed', def: false, cls: 'muted', tip: 'Official disclosure/report date. Historical rows may not include it yet.', cell: filedCellHtml }, { id: 'imported', label: 'Imported', sort: 'imported', def: false, cls: 'muted', tier: 'admin', tip: 'When Congress.Trade imported each filing.', cell: function (r) { return dateTimeCellHtml(r.imported, 'When Congress.Trade imported each filing'); } }, - { id: 'chamber', label: 'Chamber', sort: 'chamber', def: false, cls: 'muted', tip: 'House or Senate source chamber.', cell: function (r) { return esc(ownerLabel(r.chamber) || '—'); } }, + { id: 'chamber', label: 'Chamber', sort: 'chamber', def: false, cls: 'muted', tip: 'House or Senate source chamber.', cell: function (r) { return clipTextHtml(ownerLabel(r.chamber)); } }, { id: 'conf', label: 'Confidence', sort: 'conf', def: false, tier: 'admin', tip: 'Parser confidence after validation penalties.', cell: function (r) { return '~' + (r.conf * 100).toFixed(0) + '%'; } }, - { id: 'source', label: 'Source', sort: 'source', def: false, tier: 'admin', tip: 'Row provenance: primary official pipeline or historical seed import.', cell: function (r) { return '' + esc(sourceLabel(r.source)) + ''; } }, + { id: 'source', label: 'Source', sort: 'source', def: false, tier: 'admin', tip: 'Row provenance: primary official pipeline or historical seed import.', cell: function (r) { return clipTextHtml(sourceLabel(r.source), '—', sourceTitle(r.source)); } }, { id: 'latency', label: 'Latency', sort: null, def: false, cls: 'latency', tier: 'admin', tip: 'Released to seen, then seen to imported for primary rows.', cell: function (r) { return rowLatencyHtml(r); } } ]; var COL_HIDDEN_KEY = 'feed-cols-hidden-v2'; +var COL_ORDER_KEY = 'feed-cols-order-v1'; function isAdminView() { - return typeof ME !== 'undefined' && !!(ME.admin && ME.admin.allowed); + return typeof ME !== 'undefined' && !!((ME.admin && ME.admin.allowed) || hasAdminToken()); } function canUseColumn(c) { if (c.tier === 'admin') return isAdminView(); if (c.tier === 'premium') return isAdminView() || (typeof ME !== 'undefined' && isPremium()); return true; } -function availableCols() { return FEED_COLS.filter(canUseColumn); } +function loadColOrder() { try { var v = JSON.parse(localStorage.getItem(COL_ORDER_KEY)); return Array.isArray(v) ? v : []; } catch (e) { return []; } } +function saveColOrder(v) { try { localStorage.setItem(COL_ORDER_KEY, JSON.stringify(v)); } catch (e) {} } +var colOrder = loadColOrder(); +function orderedCols(cols) { + var pos = {}; + colOrder.forEach(function (id, i) { pos[id] = i; }); + return cols.slice().sort(function (a, b) { + var ai = pos[a.id], bi = pos[b.id]; + if (ai == null && bi == null) return FEED_COLS.indexOf(a) - FEED_COLS.indexOf(b); + if (ai == null) return 1; + if (bi == null) return -1; + return ai - bi; + }); +} +function chooserCols() { + return orderedCols(FEED_COLS.filter(function (c) { + if (c.lock) return false; + if (c.tier === 'admin' && !isAdminView()) return false; + return true; + })); +} +function availableCols() { return orderedCols(FEED_COLS.filter(canUseColumn)); } function defaultHidden() { return availableCols().filter(function (c) { return !c.def; }).map(function (c) { return c.id; }); } function loadHiddenCols() { try { var v = JSON.parse(localStorage.getItem(COL_HIDDEN_KEY)); return v && v.length !== undefined ? v : defaultHidden(); } catch (e) { return defaultHidden(); } } function saveHiddenCols(h) { try { localStorage.setItem(COL_HIDDEN_KEY, JSON.stringify(h)); } catch (e) {} } var hiddenCols = loadHiddenCols(); function isColVisible(id) { return hiddenCols.indexOf(id) < 0; } function visibleCols() { return availableCols().filter(function (c) { return isColVisible(c.id); }); } +function renderFeedColGroup() { + var cg = el('feedCols'); if (!cg) return; + cg.innerHTML = visibleCols().map(function (c) { return ''; }).join(''); +} +function parsePx(v) { + var n = parseFloat(v); + return Number.isFinite(n) ? n : 0; +} +function syncFeedTableWidth() { + var table = el('feedTable'); if (!table) return; + var ths = Array.prototype.slice.call(document.querySelectorAll('#feedHead th')); + var cols = Array.prototype.slice.call(document.querySelectorAll('#feedCols col')); + if (!ths.length) return; + var total = 0; + for (var i = 0; i < ths.length; i++) { + var w = parsePx(ths[i].style.width) || ths[i].offsetWidth || minColWidth(ths[i].dataset.col); + w = Math.max(minColWidth(ths[i].dataset.col), Math.round(w)); + ths[i].style.width = w + 'px'; + if (cols[i]) cols[i].style.width = w + 'px'; + total += w; + } + var wrap = table.closest ? table.closest('.table-wrap') : null; + var min = wrap ? wrap.clientWidth : 0; + table.style.width = Math.max(total, min) + 'px'; +} /* Render the header from the registry, (re)attach sort handlers, and reset the resize state so widths re-freeze for the now-visible columns. */ function renderFeedHeader() { var head = el('feedHead'); if (!head) return; + renderFeedColGroup(); head.innerHTML = visibleCols().map(function (c) { var cls = (c.sort ? 'sortable ' : '') + 'c-' + c.id; var ds = c.sort ? ' data-sort="' + c.sort + '"' : ''; @@ -2157,7 +2214,7 @@ function renderFeedHeader() { var ths = head.querySelectorAll('th.sortable'); for (var i = 0; i < ths.length; i++) { (function (th) { th.onclick = function () { setSort(th.dataset.sort); }; })(ths[i]); } // Re-init the resizable columns for the new header. - var table = el('feedTable'); if (table) table.classList.remove('resizable'); + var table = el('feedTable'); if (table) { table.classList.remove('resizable'); table.style.width = ''; } colResizeInit = false; updateSortIndicators(); } @@ -2190,17 +2247,14 @@ function renderColChooser() { var note = lockedPremium ? '
Premium enrichment
Sector, market cap, and country are available with Premium.
' : ''; - box.innerHTML = note + FEED_COLS.filter(function (c) { - if (c.lock) return false; - if (c.tier === 'admin' && !isAdminView()) return false; - return true; - }).map(function (c) { + note += '
Drag columns here to reorder the Trades table.
'; + box.innerHTML = note + chooserCols().map(function (c) { var tip = c.tip ? ' title="' + esc(c.tip) + '"' : ''; if (c.tier === 'premium' && lockedPremium) { - return ''; + return ''; } - return ''; + return ''; }).join(''); } function toggleColChooser() { @@ -2214,7 +2268,24 @@ function onColToggle(id, visible) { saveHiddenCols(hiddenCols); renderFeedHeader(); renderFeed(); } -function resetCols() { hiddenCols = defaultHidden(); saveHiddenCols(hiddenCols); renderColChooser(); renderFeedHeader(); renderFeed(); } +function moveColumn(dragId, targetId) { + if (!dragId || !targetId || dragId === targetId) return; + var ids = chooserCols().map(function (c) { return c.id; }); + ids = ids.filter(function (id) { return id !== dragId; }); + var idx = ids.indexOf(targetId); + if (idx < 0) return; + ids.splice(idx, 0, dragId); + colOrder = ids; + saveColOrder(colOrder); + renderColChooser(); renderFeedHeader(); renderFeed(); +} +function resetCols() { + hiddenCols = defaultHidden(); + colOrder = []; + saveHiddenCols(hiddenCols); + saveColOrder(colOrder); + renderColChooser(); renderFeedHeader(); renderFeed(); +} function renderFeed() { var m = el('qMember').value.toLowerCase(), t = el('qTicker').value.toUpperCase(), @@ -2251,7 +2322,7 @@ function renderFeed() { if (rows.length === 0) { body.innerHTML = stateRow(cols.length, 'No transactions match these filters.'); if (cards) cards.innerHTML = stateCards('No transactions match these filters.'); - updateFeedCountMsg(0); maybeInitResize(); return; + updateFeedCountMsg(0); maybeInitResize(); syncFeedTableWidth(); return; } body.innerHTML = rows.map(function (r) { var tds = cols.map(function (c) { @@ -2262,6 +2333,7 @@ function renderFeed() { if (cards) cards.innerHTML = rows.map(feedCardHtml).join(''); updateFeedCountMsg(rows.length); maybeInitResize(); + syncFeedTableWidth(); } /* "Showing X-Y of N" + previous/next controls for the bounded table page. */ @@ -2290,7 +2362,7 @@ function updateFeedCountMsg(shown) { } /* ---- resizable feed columns (drag the right edge of a header) ---- */ -var COL_WIDTH_KEY = 'feed-col-widths-v7'; +var COL_WIDTH_KEY = 'feed-col-widths-v8'; var colResizeInit = false; function loadColWidths() { try { return JSON.parse(localStorage.getItem(COL_WIDTH_KEY) || '{}') || {}; } catch (e) { return {}; } } function saveColWidths(w) { try { localStorage.setItem(COL_WIDTH_KEY, JSON.stringify(w)); } catch (e) {} } @@ -2312,7 +2384,7 @@ function estimatedColWidth(key, fallback, min, max) { } function minColWidth(key) { var map = { - asset: 48, + asset: 40, member: 62, amount: 56, imported: 62, @@ -2355,7 +2427,7 @@ function initColumnResize() { // compact default (Asset fits the longest name otherwise) — short entries then // show in full, long ones clip to an ellipsis, and any column stays draggable. var DEFAULT_CAP = { - asset: estimatedColWidth('asset', 54, 48, 62), + asset: estimatedColWidth('asset', 48, 40, 54), member: estimatedColWidth('member', 220, 160, 286) }; for (var i = 0; i < ths.length; i++) { @@ -2366,6 +2438,7 @@ function initColumnResize() { } table.classList.add('resizable'); for (var j = 0; j < ths.length; j++) addColResizer(ths[j]); + syncFeedTableWidth(); applyColumnWidthClasses(); } function addColResizer(th) { @@ -2377,6 +2450,7 @@ function addColResizer(th) { var startX = e.pageX, startW = th.offsetWidth; function move(ev) { th.style.width = Math.max(minColWidth(th.dataset.col), startW + (ev.pageX - startX)) + 'px'; + syncFeedTableWidth(); applyColumnWidthClasses(); } function up() { @@ -2384,6 +2458,7 @@ function addColResizer(th) { document.removeEventListener('mouseup', up); document.body.style.userSelect = ''; var w = loadColWidths(); w[th.dataset.col] = th.offsetWidth; saveColWidths(w); + syncFeedTableWidth(); applyColumnWidthClasses(); } document.addEventListener('mousemove', move); @@ -2736,7 +2811,7 @@ function loadReview() { } function loadDecisionHistory() { // API HOOK: GET /api/admin/ingestion-decisions - return fetch('/api/admin/ingestion-decisions?limit=100', { headers: adminHeaders() }) + return fetch('/api/admin/ingestion-decisions?limit=200', { headers: adminHeaders() }) .then(okOrThrow) .then(function (data) { DECISIONS = data.items || []; @@ -3305,13 +3380,13 @@ function adminHeaders(extra) { } // Turn a 401 into an actionable message instead of a bare "HTTP 401". function adminOk(r) { - if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin access box above.'); + if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin tab access box.'); if (!r.ok) throw new Error('HTTP ' + r.status); return r; } // Like adminOk but only intercepts 401 — lets the caller parse a JSON {error} body for other statuses. function admin401(r) { - if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin access box above.'); + if (r.status === 401) throw new Error('Unauthorized — paste your admin token in the Admin tab access box.'); return r; } function saveAdminToken() { @@ -3319,7 +3394,9 @@ function saveAdminToken() { try { if (v) localStorage.setItem(ADMIN_TOKEN_KEY, v); else localStorage.removeItem(ADMIN_TOKEN_KEY); } catch (e) {} el('adminTokenMsg').textContent = v ? 'Saved in this browser.' : 'Cleared.'; setTimeout(function () { el('adminTokenMsg').textContent = ''; }, 2500); + applyAdminVisibility(); renderFeedHeader(); renderColChooser(); renderFeed(); + if (v) loadReview(); loadPollConfig(); loadHealth(); loadMarketCoverage(); loadDiagnostics(); } function clearAdminToken() { @@ -3327,6 +3404,7 @@ function clearAdminToken() { if (el('adminToken')) el('adminToken').value = ''; el('adminTokenMsg').textContent = 'Cleared.'; setTimeout(function () { el('adminTokenMsg').textContent = ''; }, 2500); + applyAdminVisibility(); renderFeedHeader(); renderColChooser(); renderFeed(); } // Populate the field from storage when the Admin tab opens. @@ -4371,21 +4449,23 @@ function openTrade(row) { ? '' + esc(fmtName(row.member)) + '' : esc(fmtName(row.member)); var sideWord = row.type === 'P' ? 'Bought' : row.type === 'S' ? 'Sold' : 'Exchanged'; + var displayTicker = isScannedPdfPlaceholder(row.ticker) ? '' : (row.ticker || ''); + var displayAsset = cleanAsset(row.asset || ''); // A trade drawer leads with the TRANSACTION (kicker + amount), not the company — // the ticker/company is demoted to a non-clickable "in …" line so it can't be // mistaken for the company drawer (the ticker is intentionally NOT clickable here). - var inName = (row.ticker || row.asset) + var inName = (displayTicker || displayAsset) ? '

in ' + - (row.ticker ? '' + esc(row.ticker) + '' : '') + - (row.ticker && row.asset ? '·' : '') + - (row.asset ? '' + esc(row.asset) + '' : '') + '

' + (displayTicker ? '' + esc(displayTicker) + '' : '') + + (displayTicker && displayAsset ? '·' : '') + + (displayAsset ? '' + esc(displayAsset) + '' : '') + '

' : ''; var personCard = '
Politician
' + memberAvatarHtml(fmtName(row.member), row.photoUrl) + '
' + memberVal + '
'; - var assetLabel = row.asset || row.ticker || 'Asset unavailable'; + var assetLabel = displayAsset || displayTicker || 'Unparsed Historical Filing'; var assetCard = '
Asset
' + - tickerLogoHtml(row.ticker, assetLabel) + '
' + - (row.ticker ? '' + esc(row.ticker) + '' : '') + + tickerLogoHtml(displayTicker, assetLabel) + '
' + + (displayTicker ? '' + esc(displayTicker) + '' : '') + '' + esc(assetLabel) + '
'; var head = '
' + @@ -4454,7 +4534,8 @@ var ME = { user: null, entitlement: { premium: false, status: null, plan: null, var selectedPlan = 'monthly'; function isPremium() { return !!(ME.entitlement && ME.entitlement.premium); } -function canUseAdmin() { return !!(ME.user && ME.admin && ME.admin.allowed); } +function hasAdminToken() { return !!getAdminToken(); } +function canUseAdmin() { return !!((ME.user && ME.admin && ME.admin.allowed) || hasAdminToken()); } function updatePremiumCues() { var unlocked = isPremium() || isAdminView(); document.querySelectorAll('[data-premium-cue]').forEach(function (node) { node.hidden = unlocked; }); @@ -4517,6 +4598,7 @@ function renderAccount() { '' + '
TimeDocActionSourceReasonRows