From 1211f9bbf3188e83927f091754bd63cbfdd31bcc Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 02:10:49 -0400 Subject: [PATCH 1/3] feat(cutover): stage pipeline + milestone highlights on the operator dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-facing monitoring for the phased cutover (ENG-461 rehearsal + cutover day): operator-dashboard.ts — pure, unit-tested additions: - summarizeCashWalletCutoverStages: groups the 14-status migration machine into 10 operator stages (pending/provisioning/moving/fees/ finalizing/complete/skipped/attention/rollingBack/rolledBack) with per-status counts, in-flight/done aggregates, and percentComplete. - deriveCashWalletCutoverMilestones: diffs consecutive stage summaries into a highlights feed — run started/COMPLETE/ROLLED BACK config transitions, provisioning complete (missing-USDT reaches zero), first account in flight, first fully migrated, 25/50/75% thresholds, 100% with a reconcile reminder, attention spikes (+N failed/review) and queue-cleared, rollback started and drained. cash-wallet-cutover-dashboard.ts: - Snapshot observer appends milestones (bounded feed, each also logged via pino so kubectl logs preserves history) + 30s background poll so milestones flow with no browser open. - /api/milestones endpoint; /api/snapshot now carries stages+milestones. - UI: pipeline section (stacked stage bar + clickable stage chips that filter the account table), highlights panel (reverse-chron colored feed), run context line (state · runId · elapsed · % done), adaptive refresh (15s while a run or rollback is active, 120s idle). 9 new unit tests; tsc + eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cash-wallet-cutover/operator-dashboard.ts | 197 ++++++++++++++++++ src/scripts/cash-wallet-cutover-dashboard.ts | 171 ++++++++++++++- .../operator-dashboard-stages.spec.ts | 172 +++++++++++++++ 3 files changed, 535 insertions(+), 5 deletions(-) create mode 100644 test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index 982c6ff64..a1bfb1bf5 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -948,3 +948,200 @@ export const buildCashWalletCutoverOperatorSnapshot = async ({ reconciliation, } } + +// ── Stage pipeline + milestone highlights (operator monitoring) ───────────── +// +// Groups the 14-state migration machine into operator-facing stages and +// derives milestone events by diffing consecutive stage summaries — so the +// dashboard can show "where is the run" at a glance and a highlights feed of +// every meaningful transition (run started, provisioning done, 25/50/75/100% +// migrated, attention spikes, rollback progress). + +export type CashWalletCutoverStageKey = + | "pending" + | "provisioning" + | "moving" + | "fees" + | "finalizing" + | "complete" + | "skipped" + | "attention" + | "rollingBack" + | "rolledBack" + +export const CASH_WALLET_CUTOVER_STAGE_STATUSES: Record< + CashWalletCutoverStageKey, + (CashWalletMigrationStatus | "none")[] +> = { + pending: ["none", "not_started"], + provisioning: ["started", "provisioned"], + moving: [ + "balance_read", + "invoice_created", + "balance_move_sending", + "balance_move_sent", + "balance_move_verified", + ], + fees: [ + "fee_reimbursement_invoice_created", + "fee_reimbursement_sending", + "fee_reimbursed", + ], + finalizing: ["pointer_flipped", "legacy_zero_verified"], + complete: ["complete"], + skipped: ["skipped_already_migrated"], + attention: ["failed", "requires_operator_review"], + rollingBack: ["rollback_started"], + rolledBack: ["rolled_back"], +} + +export type CashWalletCutoverStageSummary = { + total: number + counts: Record + statusCounts: Record + inFlight: number + done: number + percentComplete: number + missingUsdtWallets: number + cutoverState: CashWalletCutoverState + runId?: string +} + +export const summarizeCashWalletCutoverStages = ({ + migrationStatuses, + cutoverState, + runId, + missingUsdtWallets, +}: { + migrationStatuses: (CashWalletMigrationStatus | "none")[] + cutoverState: CashWalletCutoverState + runId?: string + missingUsdtWallets: number +}): CashWalletCutoverStageSummary => { + const counts = Object.fromEntries( + Object.keys(CASH_WALLET_CUTOVER_STAGE_STATUSES).map((key) => [key, 0]), + ) as Record + const statusCounts: Record = {} + + const stageByStatus = new Map() + for (const [stage, statuses] of Object.entries(CASH_WALLET_CUTOVER_STAGE_STATUSES)) { + for (const status of statuses) { + stageByStatus.set(status, stage as CashWalletCutoverStageKey) + } + } + + for (const status of migrationStatuses) { + statusCounts[status] = (statusCounts[status] ?? 0) + 1 + const stage = stageByStatus.get(status) ?? "pending" + counts[stage] += 1 + } + + const inFlight = counts.provisioning + counts.moving + counts.fees + counts.finalizing + const done = counts.complete + counts.skipped + const total = migrationStatuses.length + + return { + total, + counts, + statusCounts, + inFlight, + done, + percentComplete: total > 0 ? Math.floor((done / total) * 100) : 0, + missingUsdtWallets, + cutoverState, + runId, + } +} + +export type CashWalletCutoverMilestoneKind = "ok" | "info" | "warn" | "bad" + +export type CashWalletCutoverMilestone = { + at: string + kind: CashWalletCutoverMilestoneKind + text: string +} + +const PERCENT_THRESHOLDS = [25, 50, 75] as const + +export const deriveCashWalletCutoverMilestones = ({ + previous, + current, + at, +}: { + previous?: CashWalletCutoverStageSummary + current: CashWalletCutoverStageSummary + at: string +}): CashWalletCutoverMilestone[] => { + const milestones: CashWalletCutoverMilestone[] = [] + const push = (kind: CashWalletCutoverMilestoneKind, text: string) => + milestones.push({ at, kind, text }) + + if (!previous) return milestones + + if (previous.cutoverState !== current.cutoverState) { + if (current.cutoverState === "in_progress") { + push("info", `Cutover run started${current.runId ? ` (${current.runId})` : ""}`) + } else if (current.cutoverState === "complete") { + push("ok", "Cutover config marked COMPLETE") + } else if (current.cutoverState === "rolled_back") { + push("warn", "Cutover config marked ROLLED BACK") + } else { + push("info", `Cutover state: ${previous.cutoverState} -> ${current.cutoverState}`) + } + } + + if (previous.missingUsdtWallets > 0 && current.missingUsdtWallets === 0) { + push("ok", `Provisioning complete: every account has a USDT wallet`) + } + + if (previous.inFlight === 0 && current.inFlight > 0) { + push("info", `Migration batch running (${current.inFlight} in flight)`) + } + + if (previous.counts.complete === 0 && current.counts.complete > 0) { + push("ok", "First account fully migrated") + } + + for (const threshold of PERCENT_THRESHOLDS) { + if (previous.percentComplete < threshold && current.percentComplete >= threshold) { + push("info", `${threshold}% migrated (${current.done}/${current.total})`) + } + } + + const fullyDone = + current.total > 0 && current.done === current.total && current.counts.attention === 0 + const previouslyDone = + previous.total > 0 && + previous.done === previous.total && + previous.counts.attention === 0 + if (fullyDone && !previouslyDone) { + push( + "ok", + `100% migrated (${current.done}/${current.total}) — run reconciliation and hold per runbook`, + ) + } + + if (current.counts.attention > previous.counts.attention) { + const delta = current.counts.attention - previous.counts.attention + push( + "bad", + `+${delta} account(s) need attention (failed / operator review) — total ${current.counts.attention}`, + ) + } + if (previous.counts.attention > 0 && current.counts.attention === 0) { + push("ok", "Attention queue cleared") + } + + if (previous.counts.rollingBack === 0 && current.counts.rollingBack > 0) { + push("warn", `Rollback in progress (${current.counts.rollingBack} account(s))`) + } + if ( + previous.counts.rollingBack > 0 && + current.counts.rollingBack === 0 && + current.counts.rolledBack > previous.counts.rolledBack + ) { + push("ok", `Rollback drained — ${current.counts.rolledBack} account(s) rolled back`) + } + + return milestones +} diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts index 3a3788f39..dcd0d767f 100644 --- a/src/scripts/cash-wallet-cutover-dashboard.ts +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -10,13 +10,17 @@ import { hideBin } from "yargs/helpers" import { discoverCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/discovery" import { buildCashWalletCutoverOperatorSnapshot, + CashWalletCutoverMilestone, CashWalletCutoverOperatorManifestAccount, CashWalletCutoverOperatorSnapshot, + CashWalletCutoverStageSummary, + deriveCashWalletCutoverMilestones, formatCashWalletCutoverOperatorSnapshotCsv, formatOperatorBalance, OperatorBalance, parseCashWalletCutoverOperatorManifest, refreshOperatorAccountCutoverBalanceAudit, + summarizeCashWalletCutoverStages, } from "@app/cash-wallet-cutover/operator-dashboard" import { buildCashWalletCutoverPreflightReport } from "@app/cash-wallet-cutover/preflight" import { getBalanceForWallet } from "@app/wallets" @@ -329,7 +333,26 @@ const html = ` .info { color: var(--blue); background: var(--blue-bg); } .right { text-align: right; } + .stagegrid { display: grid; grid-template-columns: 1.6fr 1fr; gap: 10px; margin-bottom: 14px; } + .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + .panel-title { font-weight: 650; margin-bottom: 10px; display: flex; justify-content: space-between; gap: 8px; align-items: baseline; } + .bar { display: flex; height: 22px; border-radius: 6px; overflow: hidden; border: 1px solid var(--line); background: #eef1f6; } + .bar div { height: 100%; transition: width .4s ease; min-width: 0; } + .chips { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 10px; } + .chip { display: inline-flex; align-items: center; gap: 6px; border: 1px solid var(--line); border-radius: 999px; padding: 3px 10px; cursor: pointer; font-size: 12px; font-weight: 600; background: #fff; } + .chip .dot { width: 9px; height: 9px; border-radius: 50%; } + .chip.active { border-color: var(--blue); box-shadow: 0 0 0 2px var(--blue-bg); } + .chip .n { color: var(--muted); font-weight: 700; } + .chip.zero { opacity: .45; } + .feed { list-style: none; margin: 0; padding: 0; max-height: 200px; overflow: auto; font-size: 13px; } + .feed li { display: flex; gap: 8px; align-items: baseline; padding: 4px 0; border-bottom: 1px dashed var(--line); } + .feed .t { color: var(--muted); font-size: 11px; white-space: nowrap; font-family: ui-monospace, Menlo, monospace; } + .feed .dot { width: 8px; height: 8px; border-radius: 50%; flex: none; align-self: center; } + .feed .dot.ok { background: var(--green); } .feed .dot.info { background: var(--blue); } + .feed .dot.warn { background: var(--yellow); } .feed .dot.bad { background: var(--red); } + @media (max-width: 1100px) { + .stagegrid { grid-template-columns: 1fr; } .summary { grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); } header { align-items: flex-start; flex-direction: column; } .table-wrap { max-height: none; } @@ -350,6 +373,17 @@ const html = `
+
+
+
Pipeline
+
+
+
+
+
Highlights
+
    +
    +
    @@ -384,6 +418,21 @@ const html = `
    ` @@ -696,6 +810,41 @@ const start = async () => { } | undefined let pending: Promise | undefined + + // Stage pipeline + milestone highlights: diff consecutive snapshots and + // keep a bounded feed. Also logged, so `kubectl logs` retains the history + // across browser sessions. + let lastStageSummary: CashWalletCutoverStageSummary | undefined + const MILESTONE_LIMIT = 300 + const milestones: CashWalletCutoverMilestone[] = [ + { + at: new Date().toISOString(), + kind: "info", + text: `Dashboard online — watching ${manifestAccounts.length} accounts`, + }, + ] + + const observeStageProgress = (result: CashWalletCutoverOperatorSnapshot) => { + const current = summarizeCashWalletCutoverStages({ + migrationStatuses: result.accounts.map((account) => account.migrationStatus), + cutoverState: result.cutover.state, + runId: result.cutover.runId, + missingUsdtWallets: result.summary.wallets.missingUsdt, + }) + const fresh = deriveCashWalletCutoverMilestones({ + previous: lastStageSummary, + current, + at: new Date().toISOString(), + }) + for (const milestone of fresh) { + milestones.push(milestone) + baseLogger.info({ milestone }, "cutover milestone") + } + if (milestones.length > MILESTONE_LIMIT) { + milestones.splice(0, milestones.length - MILESTONE_LIMIT) + } + lastStageSummary = current + } const walletCurrencies = new Map() const balanceCache = new Map() const balanceQueue: Array<{ walletId: WalletId; currency: WalletCurrency }> = [] @@ -858,6 +1007,7 @@ const start = async () => { }), }) registerSnapshotWallets(result) + observeStageProgress(result) return result } @@ -879,11 +1029,19 @@ const start = async () => { return pending } + // Keep stage/milestone tracking alive even with no browser tab open. + setInterval(() => { + snapshot(false).catch((error) => + baseLogger.warn({ error }, "cutover dashboard background poll failed"), + ) + }, 30_000) + const app = express() app.get("/", (_req, res) => res.type("html").send(html)) app.get("/api/snapshot", async (req, res) => { try { - res.json(await snapshot(req.query.refresh === "1")) + const result = await snapshot(req.query.refresh === "1") + res.json({ ...result, stages: lastStageSummary, milestones }) } catch (error) { baseLogger.error({ error }, "Cash wallet cutover dashboard snapshot failed") res.status(500).json({ @@ -946,6 +1104,9 @@ const start = async () => { }, }) }) + app.get("/api/milestones", (_req, res) => { + res.json({ stages: lastStageSummary, milestones }) + }) app.get("/api/export.csv", async (_req, res) => { try { const currentSnapshot = await snapshot(false) diff --git a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts new file mode 100644 index 000000000..34f30fb16 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts @@ -0,0 +1,172 @@ +import { + deriveCashWalletCutoverMilestones, + summarizeCashWalletCutoverStages, +} from "@app/cash-wallet-cutover/operator-dashboard" + +const AT = "2026-07-05T12:00:00.000Z" + +const summary = ( + statuses: (CashWalletMigrationStatus | "none")[], + { + state = "in_progress" as CashWalletCutoverState, + missingUsdt = 0, + runId = "run-1", + } = {}, +) => + summarizeCashWalletCutoverStages({ + migrationStatuses: statuses, + cutoverState: state, + runId, + missingUsdtWallets: missingUsdt, + }) + +describe("summarizeCashWalletCutoverStages", () => { + it("groups every migration status into a stage", () => { + const s = summary([ + "none", + "not_started", + "started", + "balance_move_sending", + "fee_reimbursed", + "pointer_flipped", + "complete", + "skipped_already_migrated", + "failed", + "requires_operator_review", + "rollback_started", + "rolled_back", + ]) + + expect(s.counts).toMatchObject({ + pending: 2, + provisioning: 1, + moving: 1, + fees: 1, + finalizing: 1, + complete: 1, + skipped: 1, + attention: 2, + rollingBack: 1, + rolledBack: 1, + }) + expect(s.inFlight).toBe(4) + expect(s.done).toBe(2) + expect(s.total).toBe(12) + }) + + it("computes percentComplete from complete+skipped", () => { + const s = summary(["complete", "skipped_already_migrated", "none", "none"]) + expect(s.percentComplete).toBe(50) + }) +}) + +describe("deriveCashWalletCutoverMilestones", () => { + it("emits nothing on the first observation", () => { + expect( + deriveCashWalletCutoverMilestones({ current: summary(["none"]), at: AT }), + ).toEqual([]) + }) + + it("announces run start, completion, and rollback state changes", () => { + const pre = summary(["none"], { state: "pre" }) + const started = deriveCashWalletCutoverMilestones({ + previous: pre, + current: summary(["none"]), + at: AT, + }) + expect(started).toEqual([ + { at: AT, kind: "info", text: "Cutover run started (run-1)" }, + ]) + + const done = deriveCashWalletCutoverMilestones({ + previous: summary(["complete"]), + current: summary(["complete"], { state: "complete" }), + at: AT, + }) + expect(done.some((m) => m.kind === "ok" && /COMPLETE/.test(m.text))).toBe(true) + + const rolled = deriveCashWalletCutoverMilestones({ + previous: summary(["rolled_back"]), + current: summary(["rolled_back"], { state: "rolled_back" }), + at: AT, + }) + expect(rolled.some((m) => m.kind === "warn" && /ROLLED BACK/.test(m.text))).toBe(true) + }) + + it("announces provisioning completion when missing USDT wallets reach zero", () => { + const out = deriveCashWalletCutoverMilestones({ + previous: summary(["none"], { missingUsdt: 5 }), + current: summary(["none"], { missingUsdt: 0 }), + at: AT, + }) + expect(out.some((m) => /Provisioning complete/.test(m.text))).toBe(true) + }) + + it("announces first in-flight, first complete, and percent thresholds", () => { + const quiet = summary(["none", "none", "none", "none"]) + const moving = deriveCashWalletCutoverMilestones({ + previous: quiet, + current: summary(["balance_move_sending", "none", "none", "none"]), + at: AT, + }) + expect(moving.some((m) => /Migration batch running/.test(m.text))).toBe(true) + + const firstDone = deriveCashWalletCutoverMilestones({ + previous: summary(["balance_move_sending", "none", "none", "none"]), + current: summary(["complete", "none", "none", "none"]), + at: AT, + }) + expect(firstDone.some((m) => /First account fully migrated/.test(m.text))).toBe(true) + expect(firstDone.some((m) => /25% migrated/.test(m.text))).toBe(true) + + const allDone = deriveCashWalletCutoverMilestones({ + previous: summary(["complete", "complete", "complete", "none"]), + current: summary(["complete", "complete", "complete", "complete"]), + at: AT, + }) + expect(allDone.some((m) => /100% migrated/.test(m.text))).toBe(true) + }) + + it("announces attention spikes and the queue clearing", () => { + const spike = deriveCashWalletCutoverMilestones({ + previous: summary(["none", "none"]), + current: summary(["failed", "requires_operator_review"]), + at: AT, + }) + expect( + spike.some( + (m) => m.kind === "bad" && /\+2 account\(s\) need attention/.test(m.text), + ), + ).toBe(true) + + const cleared = deriveCashWalletCutoverMilestones({ + previous: summary(["failed", "none"]), + current: summary(["complete", "none"]), + at: AT, + }) + expect(cleared.some((m) => /Attention queue cleared/.test(m.text))).toBe(true) + }) + + it("announces rollback start and drain", () => { + const started = deriveCashWalletCutoverMilestones({ + previous: summary(["complete"]), + current: summary(["rollback_started"]), + at: AT, + }) + expect(started.some((m) => /Rollback in progress/.test(m.text))).toBe(true) + + const drained = deriveCashWalletCutoverMilestones({ + previous: summary(["rollback_started", "rolled_back"]), + current: summary(["rolled_back", "rolled_back"]), + at: AT, + }) + expect(drained.some((m) => /Rollback drained — 2/.test(m.text))).toBe(true) + }) + + it("does not repeat milestones when nothing changes", () => { + const same = summary(["complete", "none"]) + expect( + deriveCashWalletCutoverMilestones({ previous: same, current: same, at: AT }), + ).toEqual([]) + }) +}) From 2a2c892a613c2377419def42a9c02cfec793c644 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 09:42:57 -0400 Subject: [PATCH 2/3] fix(cutover-dashboard): batch balance requests to avoid HTTP 431 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard fetched every wallet balance in a single GET whose query string carried the whole fleet's UUIDs (600+ ≈ 22KB), exceeding Node's 16KB max header size → HTTP 431, so the browser's balance hydration always failed and every balance stayed at zero (the empty 431 body's parse surfaced in Safari as "The string did not match the expected pattern"). Latent until the account set grew past ~430 wallets. Batch the requests at 60 wallet ids/request (~2KB URLs), paint incrementally as batches land, and order USD wallets first so the funded legacy balances operators care about appear before the USDT/treasury tail. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/scripts/cash-wallet-cutover-dashboard.ts | 61 ++++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts index dcd0d767f..55465cbcc 100644 --- a/src/scripts/cash-wallet-cutover-dashboard.ts +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -677,12 +677,16 @@ const html = ` renderRows() } + // USD wallets first (the funded legacy balances operators care about + // most), then USDT, then treasury — so the meaningful numbers fill in + // before the long tail. function allWalletIds() { - return snapshot.accounts.flatMap((account) => - account.usdWallets.concat(account.usdtWallets).map((wallet) => wallet.id) - ).concat((snapshot.treasury ? snapshot.treasury.accounts : []).flatMap((account) => - account.usdWallets.concat(account.usdtWallets).map((wallet) => wallet.id) - )) + const usd = snapshot.accounts.flatMap((a) => a.usdWallets.map((w) => w.id)) + const usdt = snapshot.accounts.flatMap((a) => a.usdtWallets.map((w) => w.id)) + const treasury = (snapshot.treasury ? snapshot.treasury.accounts : []).flatMap( + (a) => a.usdWallets.concat(a.usdtWallets).map((w) => w.id), + ) + return usd.concat(usdt).concat(treasury) } function mergeBalances(balances) { @@ -700,30 +704,49 @@ const html = ` } } + // Wallet IDs are batched into small GETs: one URL with the whole fleet + // (600+ UUIDs ≈ 22KB) exceeds Node's 16KB header limit and 431s, which + // stalled every balance at zero. 60 ids/request keeps URLs ~2KB. + const BALANCE_BATCH = 60 let balancePollTimer = null async function hydrateBalances(force) { if (!snapshot) return const walletIds = allWalletIds() if (walletIds.length === 0) return - const url = "/api/balances?walletIds=" + encodeURIComponent(walletIds.join(",")) + (force ? "&refresh=1" : "") - const response = await fetch(url) - const result = await response.json() - if (!response.ok) throw new Error(result.error || "Balance refresh failed") + let fresh = 0, errors = 0, loading = 0, pending = 0, active = false - mergeBalances(result.balances) - renderSummary() - renderRows() + for (let i = 0; i < walletIds.length; i += BALANCE_BATCH) { + const batch = walletIds.slice(i, i + BALANCE_BATCH) + const url = "/api/balances?walletIds=" + encodeURIComponent(batch.join(",")) + + (force ? "&refresh=1" : "") + const response = await fetch(url) + if (!response.ok) { + throw new Error("Balance refresh failed (" + response.status + ")") + } + const result = await response.json() + + mergeBalances(result.balances) + for (const balance of Object.values(result.balances)) { + if (balance.status === "fresh") fresh++ + else if (balance.status === "error") errors++ + else if (balance.status === "loading") loading++ + } + pending = result.queue.pending + active = !!result.queue.active + + // Paint incrementally so funded USD balances appear as batches land. + renderSummary() + renderRows() + $("status").textContent = "Loading balances " + Math.min(i + BALANCE_BATCH, walletIds.length) + + "/" + walletIds.length + "…" + } - const total = walletIds.length - const fresh = Object.values(result.balances).filter((balance) => balance.status === "fresh").length - const errors = Object.values(result.balances).filter((balance) => balance.status === "error").length - const loading = Object.values(result.balances).filter((balance) => balance.status === "loading").length $("status").textContent = "Updated " + new Date(snapshot.generatedAt).toLocaleTimeString() + - " | balances " + (fresh + errors) + "/" + total + - (result.queue.pending || result.queue.active ? " (" + result.queue.pending + " queued)" : "") + " | balances " + (fresh + errors) + "/" + walletIds.length + + (pending || active ? " (" + pending + " queued)" : "") - if ((loading > 0 || result.queue.pending > 0 || result.queue.active) && !balancePollTimer) { + if ((loading > 0 || pending > 0 || active) && !balancePollTimer) { balancePollTimer = setTimeout(() => { balancePollTimer = null hydrateBalances(false).catch((error) => { From e72edce4be2a51423ca137cdcba97bd48b53aec4 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 19:08:08 -0400 Subject: [PATCH 3/3] fix(cutover-dashboard): address PR #432 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 100% milestone denominator: new eligibleTotal excludes permanently- "none" accounts (already_usdt / residual / missing_*) once the run has started — they never get migration records, so done === total could never hold on real data (TEST: 315 manifest vs 297 migrations capped percentComplete at 94 and the reconciliation cue never fired). Before the run starts everything still counts. - Elapsed-run timer: startedAt is now serialized on the snapshot cutover block (it existed in the schema but was never plumbed, so the client condition was always false). - Hydration: generation guard stops the 15s reload from stacking a new multi-batch pass on a still-running one; the 2s poll chain now re-requests only wallets still loading instead of all 600 each pass. - Background poll: 30s cadence only while a run/rollback is live; idle polls back off to 5m (each rebuild is a discovery scan plus several mongo queries per account, previously 24/7 forever). - Client stage lists interpolated from CASH_WALLET_CUTOVER_STAGE_STATUSES instead of a hand-copy that silently diverges when statuses are added. - Filter contradiction: an explicit migration-status pick overrides a stale stage chip instead of ANDing to an empty table. - Milestone feed escapes text (runId is free-form operator input and this was the one unescaped innerHTML sink). - Reload chain watchdog: re-arms even if a load() never settles (the old setInterval recovered by construction; the self-chaining version died). - load() parses the body before assigning the global snapshot, so a transient 500 no longer replaces it with {error} and breaks rendering. 9 suites / 55 tests green (2 new). --- .../cash-wallet-cutover/operator-dashboard.ts | 30 ++++- src/scripts/cash-wallet-cutover-dashboard.ts | 108 +++++++++++++----- .../operator-dashboard-stages.spec.ts | 28 ++++- 3 files changed, 131 insertions(+), 35 deletions(-) diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index a1bfb1bf5..a49d5abcb 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -96,6 +96,7 @@ export type CashWalletCutoverOperatorSnapshot = { state: CashWalletCutoverState cutoverVersion: number runId?: string + startedAt?: string updatedAt?: string } preflight?: CashWalletCutoverPreflightReport @@ -912,6 +913,7 @@ export const buildCashWalletCutoverOperatorSnapshot = async ({ state: config.state, cutoverVersion: config.cutoverVersion, runId: config.runId, + startedAt: config.startedAt?.toISOString(), updatedAt: config.updatedAt?.toISOString(), }, preflight: preflightReport, @@ -997,6 +999,14 @@ export const CASH_WALLET_CUTOVER_STAGE_STATUSES: Record< export type CashWalletCutoverStageSummary = { total: number + /** + * Denominator for progress: once a run has started, accounts stuck at + * "none" (already_usdt / residual / missing_* discoveries) never get + * migration records and can never complete — counting them caps + * percentComplete below 100 forever (TEST: 315 manifest vs 297 migrations + * ⇒ ceiling of 94%). Before the run starts, everything counts. + */ + eligibleTotal: number counts: Record statusCounts: Record inFlight: number @@ -1039,14 +1049,17 @@ export const summarizeCashWalletCutoverStages = ({ const inFlight = counts.provisioning + counts.moving + counts.fees + counts.finalizing const done = counts.complete + counts.skipped const total = migrationStatuses.length + const eligibleTotal = + cutoverState === "pre" ? total : total - (statusCounts["none"] ?? 0) return { total, + eligibleTotal, counts, statusCounts, inFlight, done, - percentComplete: total > 0 ? Math.floor((done / total) * 100) : 0, + percentComplete: eligibleTotal > 0 ? Math.floor((done / eligibleTotal) * 100) : 0, missingUsdtWallets, cutoverState, runId, @@ -1104,20 +1117,25 @@ export const deriveCashWalletCutoverMilestones = ({ for (const threshold of PERCENT_THRESHOLDS) { if (previous.percentComplete < threshold && current.percentComplete >= threshold) { - push("info", `${threshold}% migrated (${current.done}/${current.total})`) + push("info", `${threshold}% migrated (${current.done}/${current.eligibleTotal})`) } } + // Against eligibleTotal, not total: permanently-"none" accounts never get + // migration records, so done === total could never hold on real data and + // the reconciliation cue would never fire. const fullyDone = - current.total > 0 && current.done === current.total && current.counts.attention === 0 + current.eligibleTotal > 0 && + current.done === current.eligibleTotal && + current.counts.attention === 0 const previouslyDone = - previous.total > 0 && - previous.done === previous.total && + previous.eligibleTotal > 0 && + previous.done === previous.eligibleTotal && previous.counts.attention === 0 if (fullyDone && !previouslyDone) { push( "ok", - `100% migrated (${current.done}/${current.total}) — run reconciliation and hold per runbook`, + `100% migrated (${current.done}/${current.eligibleTotal}) — run reconciliation and hold per runbook`, ) } diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts index 55465cbcc..ef8104143 100644 --- a/src/scripts/cash-wallet-cutover-dashboard.ts +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -11,6 +11,7 @@ import { discoverCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/disc import { buildCashWalletCutoverOperatorSnapshot, CashWalletCutoverMilestone, + CASH_WALLET_CUTOVER_STAGE_STATUSES, CashWalletCutoverOperatorManifestAccount, CashWalletCutoverOperatorSnapshot, CashWalletCutoverStageSummary, @@ -420,19 +421,23 @@ const html = ` let snapshot = null let stageFilter = "" + // Status lists interpolated from CASH_WALLET_CUTOVER_STAGE_STATUSES so a + // new state-machine status can't silently diverge between server counts + // and this client-side chip filter. + const STAGE_STATUSES = ${JSON.stringify(CASH_WALLET_CUTOVER_STAGE_STATUSES)} const STAGES = [ - ["pending", "Pending", "#98a2b3", ["none", "not_started"]], - ["provisioning", "Provisioning", "#7a5af8", ["started", "provisioned"]], - ["moving", "Moving funds", "#1f5eff", ["balance_read", "invoice_created", "balance_move_sending", "balance_move_sent", "balance_move_verified"]], - ["fees", "Fees", "#0ba5ec", ["fee_reimbursement_invoice_created", "fee_reimbursement_sending", "fee_reimbursed"]], - ["finalizing", "Finalizing", "#12b76a", ["pointer_flipped", "legacy_zero_verified"]], - ["complete", "Complete", "#16794c", ["complete"]], - ["skipped", "Skipped (USDT)", "#66c61c", ["skipped_already_migrated"]], - ["attention", "Attention", "#b42318", ["failed", "requires_operator_review"]], - ["rollingBack", "Rolling back", "#946200", ["rollback_started"]], - ["rolledBack", "Rolled back", "#667085", ["rolled_back"]], - ] - const stageStatuses = (key) => (STAGES.find((s) => s[0] === key) || [null, null, null, []])[3] + ["pending", "Pending", "#98a2b3"], + ["provisioning", "Provisioning", "#7a5af8"], + ["moving", "Moving funds", "#1f5eff"], + ["fees", "Fees", "#0ba5ec"], + ["finalizing", "Finalizing", "#12b76a"], + ["complete", "Complete", "#16794c"], + ["skipped", "Skipped (USDT)", "#66c61c"], + ["attention", "Attention", "#b42318"], + ["rollingBack", "Rolling back", "#946200"], + ["rolledBack", "Rolled back", "#667085"], + ].map(([key, label, color]) => [key, label, color, STAGE_STATUSES[key] || []]) + const stageStatuses = (key) => STAGE_STATUSES[key] || [] const $ = (id) => document.getElementById(id) const money = (cents) => "$" + (cents / 100).toFixed(2) @@ -586,7 +591,10 @@ const html = ` if (f.missingUsdt && account.usdtWallets.length > 0) return false if (f.nonzeroUsd && !usd) return false if (f.nonzeroUsdt && !usdt) return false - if (f.migration && account.migrationStatus !== f.migration) return false + // An explicit status pick overrides a stale stage chip (chip clicks + // clear the dropdown, but not vice versa — ANDing them can render an + // empty table while the chip still shows a nonzero count). + if (f.migration) return account.migrationStatus === f.migration if (stageFilter && stageStatuses(stageFilter).indexOf(account.migrationStatus) === -1) return false return true } @@ -662,9 +670,13 @@ const html = ` const feed = $("highlights") const items = (snapshot.milestones || []).slice().reverse() $("highlights-note").textContent = items.length + " event(s)" + // Milestone text embeds the free-form --run-id; escape it — this is the + // one innerHTML sink fed by operator input. + const esc = (s) => String(s).replace(/[&<>"']/g, (c) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])) feed.innerHTML = items.map((m) => - '
  1. ' + - new Date(m.at).toLocaleTimeString() + '' + m.text + '
    2. ' + '
  2. ' + + new Date(m.at).toLocaleTimeString() + '' + esc(m.text) + '
    3. ' ).join("") || '
  3. no events yet
    4. ' } @@ -709,14 +721,28 @@ const html = ` // stalled every balance at zero. 60 ids/request keeps URLs ~2KB. const BALANCE_BATCH = 60 let balancePollTimer = null - async function hydrateBalances(force) { + let hydrateGeneration = 0 + function loadingWalletIds() { + const ids = [] + const collect = (account) => account.usdWallets.concat(account.usdtWallets) + .forEach((w) => { if (!w.balance.status || w.balance.status === "loading") ids.push(w.id) }) + snapshot.accounts.forEach(collect) + if (snapshot.treasury) snapshot.treasury.accounts.forEach(collect) + return ids + } + async function hydrateBalances(force, onlyLoading) { if (!snapshot) return - const walletIds = allWalletIds() + // Generation guard: the 15s active-run reload starts a new pass while a + // previous multi-batch pass (and its 2s poll chain) may still be + // running — without this they stack and interleave renders. + const generation = ++hydrateGeneration + const walletIds = onlyLoading ? loadingWalletIds() : allWalletIds() if (walletIds.length === 0) return let fresh = 0, errors = 0, loading = 0, pending = 0, active = false for (let i = 0; i < walletIds.length; i += BALANCE_BATCH) { + if (generation !== hydrateGeneration) return // superseded by a newer pass const batch = walletIds.slice(i, i + BALANCE_BATCH) const url = "/api/balances?walletIds=" + encodeURIComponent(batch.join(",")) + (force ? "&refresh=1" : "") @@ -749,7 +775,9 @@ const html = ` if ((loading > 0 || pending > 0 || active) && !balancePollTimer) { balancePollTimer = setTimeout(() => { balancePollTimer = null - hydrateBalances(false).catch((error) => { + // Poll passes only re-request wallets still loading — not the full + // 600-wallet list every 2s while the server queue drains. + hydrateBalances(false, true).catch((error) => { $("status").textContent = error.message }) }, 2000) @@ -759,11 +787,14 @@ const html = ` async function load(force) { $("status").textContent = force ? "Refreshing..." : "Loading..." const response = await fetch("/api/snapshot" + (force ? "?refresh=1" : "")) - snapshot = await response.json() - if (!response.ok) throw new Error(snapshot.error || "Snapshot failed") + // Parse before assigning: a transient 500 must not replace the live + // snapshot with {error} and break every subsequent render. + const body = await response.json() + if (!response.ok) throw new Error(body.error || "Snapshot failed") + snapshot = body $("status").textContent = "Updated " + new Date(snapshot.generatedAt).toLocaleTimeString() render() - hydrateBalances(force).catch((error) => { + hydrateBalances(force, false).catch((error) => { $("status").textContent = error.message }) } @@ -790,9 +821,13 @@ const html = ` function scheduleReload() { if (reloadTimer) clearTimeout(reloadTimer) reloadTimer = setTimeout(() => { + // Watchdog: re-arm even if load() never settles — the server can hang + // behind a stuck snapshot build with no timeout, and a dead chain + // otherwise never recovers (the old setInterval did by construction). + const watchdog = setTimeout(scheduleReload, 60000) load(false) .catch((error) => { $("status").textContent = error.message }) - .finally(scheduleReload) + .finally(() => { clearTimeout(watchdog); scheduleReload() }) }, isActiveRun() ? 15000 : 120000) } @@ -1053,11 +1088,30 @@ const start = async () => { } // Keep stage/milestone tracking alive even with no browser tab open. - setInterval(() => { - snapshot(false).catch((error) => - baseLogger.warn({ error }, "cutover dashboard background poll failed"), - ) - }, 30_000) + // Tight cadence only while a run/rollback is live; idle polls back off — + // each rebuild is a discovery scan over every unlocked account plus + // several mongo queries per dashboard account, and pre-PR it only ran + // while a browser was actually polling. + const BACKGROUND_POLL_ACTIVE_MS = 30_000 + const BACKGROUND_POLL_IDLE_MS = 5 * 60_000 + const backgroundPoll = () => { + snapshot(false) + .catch((error) => + baseLogger.warn({ error }, "cutover dashboard background poll failed"), + ) + .finally(() => { + const active = + lastStageSummary !== undefined && + (lastStageSummary.cutoverState === "in_progress" || + lastStageSummary.inFlight > 0 || + lastStageSummary.counts.rollingBack > 0) + setTimeout( + backgroundPoll, + active ? BACKGROUND_POLL_ACTIVE_MS : BACKGROUND_POLL_IDLE_MS, + ) + }) + } + setTimeout(backgroundPoll, BACKGROUND_POLL_ACTIVE_MS) const app = express() app.get("/", (_req, res) => res.type("html").send(html)) diff --git a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts index 34f30fb16..3f3e33324 100644 --- a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts @@ -54,8 +54,21 @@ describe("summarizeCashWalletCutoverStages", () => { expect(s.total).toBe(12) }) - it("computes percentComplete from complete+skipped", () => { + it("excludes permanently-none accounts from the denominator once the run started", () => { + // "none" accounts (already_usdt / residual / missing_*) never get + // migration records — counting them caps percentComplete below 100 + // forever (TEST: 315 manifest vs 297 migrations ⇒ 94% ceiling). const s = summary(["complete", "skipped_already_migrated", "none", "none"]) + expect(s.total).toBe(4) + expect(s.eligibleTotal).toBe(2) + expect(s.percentComplete).toBe(100) + }) + + it("keeps none accounts in the denominator before the run starts", () => { + const s = summary(["complete", "skipped_already_migrated", "none", "none"], { + state: "pre" as CashWalletCutoverState, + }) + expect(s.eligibleTotal).toBe(4) expect(s.percentComplete).toBe(50) }) }) @@ -120,13 +133,24 @@ describe("deriveCashWalletCutoverMilestones", () => { expect(firstDone.some((m) => /25% migrated/.test(m.text))).toBe(true) const allDone = deriveCashWalletCutoverMilestones({ - previous: summary(["complete", "complete", "complete", "none"]), + previous: summary(["complete", "complete", "complete", "pointer_flipped"]), current: summary(["complete", "complete", "complete", "complete"]), at: AT, }) expect(allDone.some((m) => /100% migrated/.test(m.text))).toBe(true) }) + it("fires the 100% milestone even when permanently-none accounts exist", () => { + // The reconciliation cue must fire on real data, where the manifest + // always contains accounts that never get migration records. + const out = deriveCashWalletCutoverMilestones({ + previous: summary(["complete", "pointer_flipped", "none", "none"]), + current: summary(["complete", "complete", "none", "none"]), + at: AT, + }) + expect(out.some((m) => /100% migrated \(2\/2\)/.test(m.text))).toBe(true) + }) + it("announces attention spikes and the queue clearing", () => { const spike = deriveCashWalletCutoverMilestones({ previous: summary(["none", "none"]),