diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index 982c6ff64..a49d5abcb 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -96,6 +96,7 @@ export type CashWalletCutoverOperatorSnapshot = { state: CashWalletCutoverState cutoverVersion: number runId?: string + startedAt?: string updatedAt?: string } preflight?: CashWalletCutoverPreflightReport @@ -912,6 +913,7 @@ export const buildCashWalletCutoverOperatorSnapshot = async ({ state: config.state, cutoverVersion: config.cutoverVersion, runId: config.runId, + startedAt: config.startedAt?.toISOString(), updatedAt: config.updatedAt?.toISOString(), }, preflight: preflightReport, @@ -948,3 +950,216 @@ export const buildCashWalletCutoverOperatorSnapshot = async ({ reconciliation, } } + +// ── Stage pipeline + milestone highlights (operator monitoring) ───────────── +// +// Groups the 14-state migration machine into operator-facing stages and +// derives milestone events by diffing consecutive stage summaries — so the +// dashboard can show "where is the run" at a glance and a highlights feed of +// every meaningful transition (run started, provisioning done, 25/50/75/100% +// migrated, attention spikes, rollback progress). + +export type CashWalletCutoverStageKey = + | "pending" + | "provisioning" + | "moving" + | "fees" + | "finalizing" + | "complete" + | "skipped" + | "attention" + | "rollingBack" + | "rolledBack" + +export const CASH_WALLET_CUTOVER_STAGE_STATUSES: Record< + CashWalletCutoverStageKey, + (CashWalletMigrationStatus | "none")[] +> = { + pending: ["none", "not_started"], + provisioning: ["started", "provisioned"], + moving: [ + "balance_read", + "invoice_created", + "balance_move_sending", + "balance_move_sent", + "balance_move_verified", + ], + fees: [ + "fee_reimbursement_invoice_created", + "fee_reimbursement_sending", + "fee_reimbursed", + ], + finalizing: ["pointer_flipped", "legacy_zero_verified"], + complete: ["complete"], + skipped: ["skipped_already_migrated"], + attention: ["failed", "requires_operator_review"], + rollingBack: ["rollback_started"], + rolledBack: ["rolled_back"], +} + +export type CashWalletCutoverStageSummary = { + total: number + /** + * Denominator for progress: once a run has started, accounts stuck at + * "none" (already_usdt / residual / missing_* discoveries) never get + * migration records and can never complete — counting them caps + * percentComplete below 100 forever (TEST: 315 manifest vs 297 migrations + * ⇒ ceiling of 94%). Before the run starts, everything counts. + */ + eligibleTotal: number + counts: Record + statusCounts: Record + inFlight: number + done: number + percentComplete: number + missingUsdtWallets: number + cutoverState: CashWalletCutoverState + runId?: string +} + +export const summarizeCashWalletCutoverStages = ({ + migrationStatuses, + cutoverState, + runId, + missingUsdtWallets, +}: { + migrationStatuses: (CashWalletMigrationStatus | "none")[] + cutoverState: CashWalletCutoverState + runId?: string + missingUsdtWallets: number +}): CashWalletCutoverStageSummary => { + const counts = Object.fromEntries( + Object.keys(CASH_WALLET_CUTOVER_STAGE_STATUSES).map((key) => [key, 0]), + ) as Record + const statusCounts: Record = {} + + const stageByStatus = new Map() + for (const [stage, statuses] of Object.entries(CASH_WALLET_CUTOVER_STAGE_STATUSES)) { + for (const status of statuses) { + stageByStatus.set(status, stage as CashWalletCutoverStageKey) + } + } + + for (const status of migrationStatuses) { + statusCounts[status] = (statusCounts[status] ?? 0) + 1 + const stage = stageByStatus.get(status) ?? "pending" + counts[stage] += 1 + } + + const inFlight = counts.provisioning + counts.moving + counts.fees + counts.finalizing + const done = counts.complete + counts.skipped + const total = migrationStatuses.length + const eligibleTotal = + cutoverState === "pre" ? total : total - (statusCounts["none"] ?? 0) + + return { + total, + eligibleTotal, + counts, + statusCounts, + inFlight, + done, + percentComplete: eligibleTotal > 0 ? Math.floor((done / eligibleTotal) * 100) : 0, + missingUsdtWallets, + cutoverState, + runId, + } +} + +export type CashWalletCutoverMilestoneKind = "ok" | "info" | "warn" | "bad" + +export type CashWalletCutoverMilestone = { + at: string + kind: CashWalletCutoverMilestoneKind + text: string +} + +const PERCENT_THRESHOLDS = [25, 50, 75] as const + +export const deriveCashWalletCutoverMilestones = ({ + previous, + current, + at, +}: { + previous?: CashWalletCutoverStageSummary + current: CashWalletCutoverStageSummary + at: string +}): CashWalletCutoverMilestone[] => { + const milestones: CashWalletCutoverMilestone[] = [] + const push = (kind: CashWalletCutoverMilestoneKind, text: string) => + milestones.push({ at, kind, text }) + + if (!previous) return milestones + + if (previous.cutoverState !== current.cutoverState) { + if (current.cutoverState === "in_progress") { + push("info", `Cutover run started${current.runId ? ` (${current.runId})` : ""}`) + } else if (current.cutoverState === "complete") { + push("ok", "Cutover config marked COMPLETE") + } else if (current.cutoverState === "rolled_back") { + push("warn", "Cutover config marked ROLLED BACK") + } else { + push("info", `Cutover state: ${previous.cutoverState} -> ${current.cutoverState}`) + } + } + + if (previous.missingUsdtWallets > 0 && current.missingUsdtWallets === 0) { + push("ok", `Provisioning complete: every account has a USDT wallet`) + } + + if (previous.inFlight === 0 && current.inFlight > 0) { + push("info", `Migration batch running (${current.inFlight} in flight)`) + } + + if (previous.counts.complete === 0 && current.counts.complete > 0) { + push("ok", "First account fully migrated") + } + + for (const threshold of PERCENT_THRESHOLDS) { + if (previous.percentComplete < threshold && current.percentComplete >= threshold) { + push("info", `${threshold}% migrated (${current.done}/${current.eligibleTotal})`) + } + } + + // Against eligibleTotal, not total: permanently-"none" accounts never get + // migration records, so done === total could never hold on real data and + // the reconciliation cue would never fire. + const fullyDone = + current.eligibleTotal > 0 && + current.done === current.eligibleTotal && + current.counts.attention === 0 + const previouslyDone = + previous.eligibleTotal > 0 && + previous.done === previous.eligibleTotal && + previous.counts.attention === 0 + if (fullyDone && !previouslyDone) { + push( + "ok", + `100% migrated (${current.done}/${current.eligibleTotal}) — run reconciliation and hold per runbook`, + ) + } + + if (current.counts.attention > previous.counts.attention) { + const delta = current.counts.attention - previous.counts.attention + push( + "bad", + `+${delta} account(s) need attention (failed / operator review) — total ${current.counts.attention}`, + ) + } + if (previous.counts.attention > 0 && current.counts.attention === 0) { + push("ok", "Attention queue cleared") + } + + if (previous.counts.rollingBack === 0 && current.counts.rollingBack > 0) { + push("warn", `Rollback in progress (${current.counts.rollingBack} account(s))`) + } + if ( + previous.counts.rollingBack > 0 && + current.counts.rollingBack === 0 && + current.counts.rolledBack > previous.counts.rolledBack + ) { + push("ok", `Rollback drained — ${current.counts.rolledBack} account(s) rolled back`) + } + + return milestones +} diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts index 3a3788f39..ef8104143 100644 --- a/src/scripts/cash-wallet-cutover-dashboard.ts +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -10,13 +10,18 @@ import { hideBin } from "yargs/helpers" import { discoverCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/discovery" import { buildCashWalletCutoverOperatorSnapshot, + CashWalletCutoverMilestone, + CASH_WALLET_CUTOVER_STAGE_STATUSES, CashWalletCutoverOperatorManifestAccount, CashWalletCutoverOperatorSnapshot, + CashWalletCutoverStageSummary, + deriveCashWalletCutoverMilestones, formatCashWalletCutoverOperatorSnapshotCsv, formatOperatorBalance, OperatorBalance, parseCashWalletCutoverOperatorManifest, refreshOperatorAccountCutoverBalanceAudit, + summarizeCashWalletCutoverStages, } from "@app/cash-wallet-cutover/operator-dashboard" import { buildCashWalletCutoverPreflightReport } from "@app/cash-wallet-cutover/preflight" import { getBalanceForWallet } from "@app/wallets" @@ -329,7 +334,26 @@ const html = ` .info { color: var(--blue); background: var(--blue-bg); } .right { text-align: right; } + .stagegrid { display: grid; grid-template-columns: 1.6fr 1fr; gap: 10px; margin-bottom: 14px; } + .panel { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } + .panel-title { font-weight: 650; margin-bottom: 10px; display: flex; justify-content: space-between; gap: 8px; align-items: baseline; } + .bar { display: flex; height: 22px; border-radius: 6px; overflow: hidden; border: 1px solid var(--line); background: #eef1f6; } + .bar div { height: 100%; transition: width .4s ease; min-width: 0; } + .chips { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 10px; } + .chip { display: inline-flex; align-items: center; gap: 6px; border: 1px solid var(--line); border-radius: 999px; padding: 3px 10px; cursor: pointer; font-size: 12px; font-weight: 600; background: #fff; } + .chip .dot { width: 9px; height: 9px; border-radius: 50%; } + .chip.active { border-color: var(--blue); box-shadow: 0 0 0 2px var(--blue-bg); } + .chip .n { color: var(--muted); font-weight: 700; } + .chip.zero { opacity: .45; } + .feed { list-style: none; margin: 0; padding: 0; max-height: 200px; overflow: auto; font-size: 13px; } + .feed li { display: flex; gap: 8px; align-items: baseline; padding: 4px 0; border-bottom: 1px dashed var(--line); } + .feed .t { color: var(--muted); font-size: 11px; white-space: nowrap; font-family: ui-monospace, Menlo, monospace; } + .feed .dot { width: 8px; height: 8px; border-radius: 50%; flex: none; align-self: center; } + .feed .dot.ok { background: var(--green); } .feed .dot.info { background: var(--blue); } + .feed .dot.warn { background: var(--yellow); } .feed .dot.bad { background: var(--red); } + @media (max-width: 1100px) { + .stagegrid { grid-template-columns: 1fr; } .summary { grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); } header { align-items: flex-start; flex-direction: column; } .table-wrap { max-height: none; } @@ -350,6 +374,17 @@ const html = `
+
+
+
Pipeline
+
+
+
+
+
Highlights
+
    +
    +
    @@ -384,6 +419,25 @@ const html = `
    ` @@ -696,6 +868,41 @@ const start = async () => { } | undefined let pending: Promise | undefined + + // Stage pipeline + milestone highlights: diff consecutive snapshots and + // keep a bounded feed. Also logged, so `kubectl logs` retains the history + // across browser sessions. + let lastStageSummary: CashWalletCutoverStageSummary | undefined + const MILESTONE_LIMIT = 300 + const milestones: CashWalletCutoverMilestone[] = [ + { + at: new Date().toISOString(), + kind: "info", + text: `Dashboard online — watching ${manifestAccounts.length} accounts`, + }, + ] + + const observeStageProgress = (result: CashWalletCutoverOperatorSnapshot) => { + const current = summarizeCashWalletCutoverStages({ + migrationStatuses: result.accounts.map((account) => account.migrationStatus), + cutoverState: result.cutover.state, + runId: result.cutover.runId, + missingUsdtWallets: result.summary.wallets.missingUsdt, + }) + const fresh = deriveCashWalletCutoverMilestones({ + previous: lastStageSummary, + current, + at: new Date().toISOString(), + }) + for (const milestone of fresh) { + milestones.push(milestone) + baseLogger.info({ milestone }, "cutover milestone") + } + if (milestones.length > MILESTONE_LIMIT) { + milestones.splice(0, milestones.length - MILESTONE_LIMIT) + } + lastStageSummary = current + } const walletCurrencies = new Map() const balanceCache = new Map() const balanceQueue: Array<{ walletId: WalletId; currency: WalletCurrency }> = [] @@ -858,6 +1065,7 @@ const start = async () => { }), }) registerSnapshotWallets(result) + observeStageProgress(result) return result } @@ -879,11 +1087,38 @@ const start = async () => { return pending } + // Keep stage/milestone tracking alive even with no browser tab open. + // Tight cadence only while a run/rollback is live; idle polls back off — + // each rebuild is a discovery scan over every unlocked account plus + // several mongo queries per dashboard account, and pre-PR it only ran + // while a browser was actually polling. + const BACKGROUND_POLL_ACTIVE_MS = 30_000 + const BACKGROUND_POLL_IDLE_MS = 5 * 60_000 + const backgroundPoll = () => { + snapshot(false) + .catch((error) => + baseLogger.warn({ error }, "cutover dashboard background poll failed"), + ) + .finally(() => { + const active = + lastStageSummary !== undefined && + (lastStageSummary.cutoverState === "in_progress" || + lastStageSummary.inFlight > 0 || + lastStageSummary.counts.rollingBack > 0) + setTimeout( + backgroundPoll, + active ? BACKGROUND_POLL_ACTIVE_MS : BACKGROUND_POLL_IDLE_MS, + ) + }) + } + setTimeout(backgroundPoll, BACKGROUND_POLL_ACTIVE_MS) + const app = express() app.get("/", (_req, res) => res.type("html").send(html)) app.get("/api/snapshot", async (req, res) => { try { - res.json(await snapshot(req.query.refresh === "1")) + const result = await snapshot(req.query.refresh === "1") + res.json({ ...result, stages: lastStageSummary, milestones }) } catch (error) { baseLogger.error({ error }, "Cash wallet cutover dashboard snapshot failed") res.status(500).json({ @@ -946,6 +1181,9 @@ const start = async () => { }, }) }) + app.get("/api/milestones", (_req, res) => { + res.json({ stages: lastStageSummary, milestones }) + }) app.get("/api/export.csv", async (_req, res) => { try { const currentSnapshot = await snapshot(false) diff --git a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts new file mode 100644 index 000000000..3f3e33324 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard-stages.spec.ts @@ -0,0 +1,196 @@ +import { + deriveCashWalletCutoverMilestones, + summarizeCashWalletCutoverStages, +} from "@app/cash-wallet-cutover/operator-dashboard" + +const AT = "2026-07-05T12:00:00.000Z" + +const summary = ( + statuses: (CashWalletMigrationStatus | "none")[], + { + state = "in_progress" as CashWalletCutoverState, + missingUsdt = 0, + runId = "run-1", + } = {}, +) => + summarizeCashWalletCutoverStages({ + migrationStatuses: statuses, + cutoverState: state, + runId, + missingUsdtWallets: missingUsdt, + }) + +describe("summarizeCashWalletCutoverStages", () => { + it("groups every migration status into a stage", () => { + const s = summary([ + "none", + "not_started", + "started", + "balance_move_sending", + "fee_reimbursed", + "pointer_flipped", + "complete", + "skipped_already_migrated", + "failed", + "requires_operator_review", + "rollback_started", + "rolled_back", + ]) + + expect(s.counts).toMatchObject({ + pending: 2, + provisioning: 1, + moving: 1, + fees: 1, + finalizing: 1, + complete: 1, + skipped: 1, + attention: 2, + rollingBack: 1, + rolledBack: 1, + }) + expect(s.inFlight).toBe(4) + expect(s.done).toBe(2) + expect(s.total).toBe(12) + }) + + it("excludes permanently-none accounts from the denominator once the run started", () => { + // "none" accounts (already_usdt / residual / missing_*) never get + // migration records — counting them caps percentComplete below 100 + // forever (TEST: 315 manifest vs 297 migrations ⇒ 94% ceiling). + const s = summary(["complete", "skipped_already_migrated", "none", "none"]) + expect(s.total).toBe(4) + expect(s.eligibleTotal).toBe(2) + expect(s.percentComplete).toBe(100) + }) + + it("keeps none accounts in the denominator before the run starts", () => { + const s = summary(["complete", "skipped_already_migrated", "none", "none"], { + state: "pre" as CashWalletCutoverState, + }) + expect(s.eligibleTotal).toBe(4) + expect(s.percentComplete).toBe(50) + }) +}) + +describe("deriveCashWalletCutoverMilestones", () => { + it("emits nothing on the first observation", () => { + expect( + deriveCashWalletCutoverMilestones({ current: summary(["none"]), at: AT }), + ).toEqual([]) + }) + + it("announces run start, completion, and rollback state changes", () => { + const pre = summary(["none"], { state: "pre" }) + const started = deriveCashWalletCutoverMilestones({ + previous: pre, + current: summary(["none"]), + at: AT, + }) + expect(started).toEqual([ + { at: AT, kind: "info", text: "Cutover run started (run-1)" }, + ]) + + const done = deriveCashWalletCutoverMilestones({ + previous: summary(["complete"]), + current: summary(["complete"], { state: "complete" }), + at: AT, + }) + expect(done.some((m) => m.kind === "ok" && /COMPLETE/.test(m.text))).toBe(true) + + const rolled = deriveCashWalletCutoverMilestones({ + previous: summary(["rolled_back"]), + current: summary(["rolled_back"], { state: "rolled_back" }), + at: AT, + }) + expect(rolled.some((m) => m.kind === "warn" && /ROLLED BACK/.test(m.text))).toBe(true) + }) + + it("announces provisioning completion when missing USDT wallets reach zero", () => { + const out = deriveCashWalletCutoverMilestones({ + previous: summary(["none"], { missingUsdt: 5 }), + current: summary(["none"], { missingUsdt: 0 }), + at: AT, + }) + expect(out.some((m) => /Provisioning complete/.test(m.text))).toBe(true) + }) + + it("announces first in-flight, first complete, and percent thresholds", () => { + const quiet = summary(["none", "none", "none", "none"]) + const moving = deriveCashWalletCutoverMilestones({ + previous: quiet, + current: summary(["balance_move_sending", "none", "none", "none"]), + at: AT, + }) + expect(moving.some((m) => /Migration batch running/.test(m.text))).toBe(true) + + const firstDone = deriveCashWalletCutoverMilestones({ + previous: summary(["balance_move_sending", "none", "none", "none"]), + current: summary(["complete", "none", "none", "none"]), + at: AT, + }) + expect(firstDone.some((m) => /First account fully migrated/.test(m.text))).toBe(true) + expect(firstDone.some((m) => /25% migrated/.test(m.text))).toBe(true) + + const allDone = deriveCashWalletCutoverMilestones({ + previous: summary(["complete", "complete", "complete", "pointer_flipped"]), + current: summary(["complete", "complete", "complete", "complete"]), + at: AT, + }) + expect(allDone.some((m) => /100% migrated/.test(m.text))).toBe(true) + }) + + it("fires the 100% milestone even when permanently-none accounts exist", () => { + // The reconciliation cue must fire on real data, where the manifest + // always contains accounts that never get migration records. + const out = deriveCashWalletCutoverMilestones({ + previous: summary(["complete", "pointer_flipped", "none", "none"]), + current: summary(["complete", "complete", "none", "none"]), + at: AT, + }) + expect(out.some((m) => /100% migrated \(2\/2\)/.test(m.text))).toBe(true) + }) + + it("announces attention spikes and the queue clearing", () => { + const spike = deriveCashWalletCutoverMilestones({ + previous: summary(["none", "none"]), + current: summary(["failed", "requires_operator_review"]), + at: AT, + }) + expect( + spike.some( + (m) => m.kind === "bad" && /\+2 account\(s\) need attention/.test(m.text), + ), + ).toBe(true) + + const cleared = deriveCashWalletCutoverMilestones({ + previous: summary(["failed", "none"]), + current: summary(["complete", "none"]), + at: AT, + }) + expect(cleared.some((m) => /Attention queue cleared/.test(m.text))).toBe(true) + }) + + it("announces rollback start and drain", () => { + const started = deriveCashWalletCutoverMilestones({ + previous: summary(["complete"]), + current: summary(["rollback_started"]), + at: AT, + }) + expect(started.some((m) => /Rollback in progress/.test(m.text))).toBe(true) + + const drained = deriveCashWalletCutoverMilestones({ + previous: summary(["rollback_started", "rolled_back"]), + current: summary(["rolled_back", "rolled_back"]), + at: AT, + }) + expect(drained.some((m) => /Rollback drained — 2/.test(m.text))).toBe(true) + }) + + it("does not repeat milestones when nothing changes", () => { + const same = summary(["complete", "none"]) + expect( + deriveCashWalletCutoverMilestones({ previous: same, current: same, at: AT }), + ).toEqual([]) + }) +})