From fa953942d783a58886c61e00e93b796068958c52 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 15:37:35 -0400 Subject: [PATCH 1/7] fix(cutover): reverse spendable USDT balance on rollback, not the target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rollback reverse-payment paid back destinationAmountUsdtMicros (the forward target), but the USDT wallet legitimately holds slightly less: the un-reimbursed forward routing fee plus the balance read rounding up (e.g. 0.443691959514 read as 443692 micros). Paying the target verbatim overspends the wallet by a few hundred micros and IBEX rejects it with 400 Bad Request — leaving the account pointer-reverted but funds stranded. Fix: reverse min(spendable, target) using a new floored balance read (readDestinationSpendableUsdtMicros), so the reverse amount never exceeds what's actually spendable. Fail closed only when short by more than the existing 1-cent tolerance — the signature of genuine post-cutover spend; within tolerance, reverse the full spendable balance (user made whole minus sub-cent dust). Validated end-to-end on TEST against real IBEX (ENG-401): pointer restored -> reverse payment succeeds -> rolled_back, funds returned to legacy USD. Refs ENG-401 --- .../cash-wallet-cutover/rollback-worker.ts | 32 ++++++++++++----- .../cash-wallet-cutover/runtime-services.ts | 18 ++++++++++ .../rollback-worker.spec.ts | 36 ++++++++++++++++++- 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/app/cash-wallet-cutover/rollback-worker.ts b/src/app/cash-wallet-cutover/rollback-worker.ts index 03046e721..310a64070 100644 --- a/src/app/cash-wallet-cutover/rollback-worker.ts +++ b/src/app/cash-wallet-cutover/rollback-worker.ts @@ -40,6 +40,9 @@ type CashWalletRollbackServices = { readDestinationBalanceUsdtMicros( migration: CashWalletMigration, ): Promise + readDestinationSpendableUsdtMicros( + migration: CashWalletMigration, + ): Promise } invoiceService: { createNoAmountInvoice(args: { @@ -194,17 +197,28 @@ export const executeCashWalletMigrationRollbackStep = async ({ } if (current.destinationAmountUsdtMicros !== "0") { - // Fail closed: if the user transacted on the USDT wallet since the - // forward move, reversing the full amount is not possible without - // operator judgment. - const usdtBalance = - await services.balanceReader.readDestinationBalanceUsdtMicros(current) - if (usdtBalance instanceof Error) return usdtBalance - if (BigInt(usdtBalance) < BigInt(current.destinationAmountUsdtMicros)) { + // Reverse the ACTUAL spendable balance (floored), capped at the forward + // amount — never the raw target. The USDT wallet legitimately holds + // slightly less than target: the un-reimbursed forward routing fee plus + // sub-micro rounding. Paying the target verbatim overspends → IBEX 400 + // (ENG-401). Reversing the spendable balance makes the user whole minus + // that sub-cent dust. Fail closed only when the wallet is short by MORE + // than tolerance — the signature of the user having spent USDT + // post-cutover, which needs operator judgment. + const target = BigInt(current.destinationAmountUsdtMicros) + const spendableStr = + await services.balanceReader.readDestinationSpendableUsdtMicros(current) + if (spendableStr instanceof Error) return spendableStr + const spendable = BigInt(spendableStr) + + const reverseAmount = spendable < target ? spendable : target + const shortfall = target - reverseAmount + if (shortfall > ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS) { return new CashWalletMigrationFailedError( - `USDT balance ${usdtBalance} no longer covers reverse amount ${current.destinationAmountUsdtMicros}`, + `USDT balance ${spendableStr} is short of reverse target ${target} by ${shortfall} micros (> tolerance) — likely spent post-cutover`, ) } + const reverseAmountStr = reverseAmount.toString() if ( current.rollbackInvoicePaymentRequest === undefined || @@ -240,7 +254,7 @@ export const executeCashWalletMigrationRollbackStep = async ({ const payment = await services.paymentService.payInvoice({ senderWalletId: current.destinationUsdtWalletId, paymentRequest: current.rollbackInvoicePaymentRequest, - senderAmountUsdtMicros: current.destinationAmountUsdtMicros, + senderAmountUsdtMicros: reverseAmountStr, }) if (payment instanceof Error) return payment diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index abf75e1f0..cd6a1b71e 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -222,6 +222,24 @@ export const createCashWalletMigrationRuntimeServices = ( if (account instanceof Error) return account return ibexMajorUnitsToUsdtMicros(account.balance ?? 0) }, + // Rollback (ENG-401): the FLOORED spendable USDT balance. The rounded + // read above can round UP past what's actually spendable (e.g. balance + // 0.443691959514 → 443692 micros), so paying it back verbatim overspends + // and IBEX 400s. Flooring guarantees the reverse amount never exceeds the + // real balance. + readDestinationSpendableUsdtMicros: async ( + migration: CashWalletMigration, + ): Promise => { + const account = await rawAccountDetails( + migration.destinationUsdtWalletId as IbexAccountId, + ) + if (account instanceof Error) return account + return decimalToScaledInteger({ + amount: account.balance ?? 0, + scale: 6, + round: false, + }) + }, }, invoiceService: { createInvoice: ({ diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts index 219cfea93..3d3775213 100644 --- a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts @@ -45,10 +45,12 @@ const statefulRepo = (initial: CashWalletMigration) => { const services = ({ defaultWalletId = "legacy-wallet-id" as WalletId, usdtBalanceMicros = "0", + spendableUsdtMicros, legacyBalanceUsdCents = "0", }: { defaultWalletId?: WalletId usdtBalanceMicros?: string + spendableUsdtMicros?: string legacyBalanceUsdCents?: string } = {}) => ({ now: () => NOW, @@ -63,6 +65,9 @@ const services = ({ balanceReader: { readSourceBalanceUsdCents: jest.fn().mockResolvedValue(legacyBalanceUsdCents), readDestinationBalanceUsdtMicros: jest.fn().mockResolvedValue(usdtBalanceMicros), + readDestinationSpendableUsdtMicros: jest + .fn() + .mockResolvedValue(spendableUsdtMicros ?? usdtBalanceMicros), }, invoiceService: { createNoAmountInvoice: jest.fn().mockResolvedValue(invoice), @@ -224,6 +229,35 @@ describe("rollback executor", () => { }) }) + it("reverses the SPENDABLE balance (not the target) when short by only dust (ENG-401)", async () => { + // The wallet holds target minus the un-reimbursed forward fee + rounding + // (well within the 1-cent tolerance). Reverse the actual spendable amount, + // never the target — paying the target would overspend and IBEX 400s. + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + } + const svc = services({ + spendableUsdtMicros: "1499996473", // 3527 micros short = forward fee + legacyBalanceUsdCents: "150000", // whole again after the reverse pay + }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(svc.paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: migration.destinationUsdtWalletId, + paymentRequest: invoice.paymentRequest, + senderAmountUsdtMicros: "1499996473", // the spendable balance, not 1500000000 + }) + expect(result).toMatchObject({ status: "rolled_back" }) + }) + it("fails closed when the USDT balance no longer covers the reverse amount", async () => { const migration = { ...baseMigration, @@ -231,7 +265,7 @@ describe("rollback executor", () => { sourceBalanceUsdCents: "150000", destinationAmountUsdtMicros: "1500000000", } - const svc = services({ usdtBalanceMicros: "900000000" }) // user spent funds + const svc = services({ spendableUsdtMicros: "900000000" }) // user spent funds const result = await executeCashWalletMigrationRollbackStep({ migration, From 4513ff94c32cd8db077dfb4b382c0c5f41a6f085 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 15:51:33 -0400 Subject: [PATCH 2/7] fix(cutover): retry IBEX 429s on balance reads + default run-batch throttle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unthrottled run-batch (--limit 300, no delay) mass-failed 233/297 accounts in the ENG-461 TEST rehearsal with "FetchError: Too Many Requests". Two gaps: 1. Balance reads (getRawAccountDetails) were the one IBEX call path not wrapped in withIbexRateLimitRetry — and the wrapper only handled RETURNED errors, while the IBEX client surfaces 429s on some paths as thrown FetchError rejections. Now every account-details read (balance readers, balance verifier, fee service, rollback shortfall reader) goes through the retry, and the wrapper retries thrown rate-limit rejections as well as returned ones. 2. run-batch defaulted to --step-delay-ms 0 (unthrottled). Default is now 2000ms (~30 accounts/min, the empirically safe IBEX rate from the rehearsal). Pass --step-delay-ms 0 explicitly to disable. Refs ENG-483 --- .../cash-wallet-cutover/runtime-services.ts | 33 +++++-- src/scripts/cash-wallet-cutover.ts | 5 +- .../runtime-services.spec.ts | 87 +++++++++++++++++++ 3 files changed, 117 insertions(+), 8 deletions(-) create mode 100644 test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index cd6a1b71e..2b33c78f2 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -139,7 +139,18 @@ const withIbexRateLimitRetry = async ({ sleepFn: SleepFn }): Promise => { for (let attempt = 1; ; attempt += 1) { - const result = await operation() + // The IBEX client surfaces rate limits both ways: some call paths return + // an IbexError, others REJECT with a raw FetchError ("Too Many Requests"). + // Retry both — a thrown 429 killed 233 accounts in the ENG-461 rehearsal. + let result: T | Error + try { + result = await operation() + } catch (thrown) { + const error = thrown instanceof Error ? thrown : new Error(String(thrown)) + if (!isIbexRateLimitError(error) || attempt >= maxAttempts) throw thrown + await sleepFn(retryDelayMs) + continue + } if ( !(result instanceof Error) || @@ -171,6 +182,14 @@ export const createCashWalletMigrationRuntimeServices = ( retryDelayMs: deps.rateLimitRetryDelayMs ?? CUTOVER_IBEX_RATE_LIMIT_RETRY_DELAY_MS, sleepFn: deps.sleep ?? sleep, } + // ENG-483: balance reads were the one IBEX call path NOT retried on 429 — + // an unthrottled batch mass-failed 233 accounts at the balance_read step. + // Every account-details read goes through the same retry as invoice/payment. + const readRawAccountDetails = (accountId: IbexAccountId) => + withIbexRateLimitRetry({ + ...rateLimitRetry, + operation: () => rawAccountDetails(accountId), + }) return { now: deps.now ?? (() => new Date()), @@ -207,7 +226,7 @@ export const createCashWalletMigrationRuntimeServices = ( readSourceBalanceUsdCents: async ( migration: CashWalletMigration, ): Promise => { - const account = await rawAccountDetails( + const account = await readRawAccountDetails( migration.legacyUsdWalletId as IbexAccountId, ) if (account instanceof Error) return account @@ -216,7 +235,7 @@ export const createCashWalletMigrationRuntimeServices = ( readDestinationBalanceUsdtMicros: async ( migration: CashWalletMigration, ): Promise => { - const account = await rawAccountDetails( + const account = await readRawAccountDetails( migration.destinationUsdtWalletId as IbexAccountId, ) if (account instanceof Error) return account @@ -230,7 +249,7 @@ export const createCashWalletMigrationRuntimeServices = ( readDestinationSpendableUsdtMicros: async ( migration: CashWalletMigration, ): Promise => { - const account = await rawAccountDetails( + const account = await readRawAccountDetails( migration.destinationUsdtWalletId as IbexAccountId, ) if (account instanceof Error) return account @@ -333,7 +352,7 @@ export const createCashWalletMigrationRuntimeServices = ( }: { legacyUsdWalletId: WalletId }): Promise => { - const account = await rawAccountDetails(legacyUsdWalletId as IbexAccountId) + const account = await readRawAccountDetails(legacyUsdWalletId as IbexAccountId) if (account instanceof Error) return account if (!hasOnlySubMicroMajorUnitDust(account.balance)) { return new CashWalletMigrationFailedError("Legacy USD wallet is not zero") @@ -363,7 +382,7 @@ export const createCashWalletMigrationRuntimeServices = ( ) } - const currentAccount = await rawAccountDetails( + const currentAccount = await readRawAccountDetails( migration.destinationUsdtWalletId as IbexAccountId, ) if (currentAccount instanceof Error) return currentAccount @@ -408,7 +427,7 @@ export const createCashWalletMigrationRuntimeServices = ( }: { legacyUsdWalletId: WalletId }): Promise => { - const account = await rawAccountDetails(legacyUsdWalletId as IbexAccountId) + const account = await readRawAccountDetails(legacyUsdWalletId as IbexAccountId) if (account instanceof Error) return account if (!hasOnlySubMicroMajorUnitDust(account.balance)) { return new CashWalletMigrationFailedError("Legacy USD wallet is not zero") diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 1d5762f52..9c1806903 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -39,7 +39,10 @@ const args = yargs(hideBin(process.argv)) .option("operator", { type: "string", default: "unknown" }) .option("worker-id", { type: "string", default: `worker-${process.pid}` }) .option("limit", { type: "number", default: 25 }) - .option("step-delay-ms", { type: "number", default: 0 }) + // ENG-483: default throttle ≈30 accounts/min — the empirically safe IBEX + // rate from the ENG-461 rehearsal (0 = unthrottled mass-failed 233 accounts + // on 429s). Pass --step-delay-ms 0 explicitly to disable. + .option("step-delay-ms", { type: "number", default: 2_000 }) .option("provision-limit", { type: "number" }) .option("provision-delay-ms", { type: "number", default: 12_500 }) .option("provision-retry-delay-ms", { type: "number", default: 60_000 }) diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts new file mode 100644 index 000000000..731b50b69 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -0,0 +1,87 @@ +import { createCashWalletMigrationRuntimeServices } from "@app/cash-wallet-cutover/runtime-services" + +const migration = { + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, +} as CashWalletMigration + +const rateLimitFetchError = () => new Error("FetchError: Too Many Requests") + +describe("cutover runtime services — IBEX rate-limit retry (ENG-483)", () => { + it("retries balance reads when the read REJECTS with a 429 FetchError", async () => { + const sleeps: number[] = [] + const getRawAccountDetails = jest + .fn() + .mockRejectedValueOnce(rateLimitFetchError()) + .mockRejectedValueOnce(rateLimitFetchError()) + .mockResolvedValue({ balance: 12.5 }) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 7, + sleep: async (ms) => { + sleeps.push(ms) + }, + }) + + const result = await services.balanceReader.readSourceBalanceUsdCents(migration) + + expect(result).toBe("1250") + expect(getRawAccountDetails).toHaveBeenCalledTimes(3) + expect(sleeps).toEqual([7, 7]) + }) + + it("retries balance reads when the read RETURNS a rate-limit error", async () => { + const getRawAccountDetails = jest + .fn() + .mockResolvedValueOnce(new Error("Too Many Requests")) + .mockResolvedValue({ balance: 3 }) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + const result = + await services.balanceReader.readDestinationBalanceUsdtMicros(migration) + + expect(result).toBe("3000000") + expect(getRawAccountDetails).toHaveBeenCalledTimes(2) + }) + + it("gives up after maxAttempts and rethrows the final 429", async () => { + const getRawAccountDetails = jest.fn().mockRejectedValue(rateLimitFetchError()) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + maxRateLimitAttempts: 3, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + await expect( + services.balanceReader.readSourceBalanceUsdCents(migration), + ).rejects.toThrow("Too Many Requests") + expect(getRawAccountDetails).toHaveBeenCalledTimes(3) + }) + + it("does not retry non-rate-limit rejections", async () => { + const getRawAccountDetails = jest + .fn() + .mockRejectedValue(new Error("FetchError: Not Found")) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + await expect( + services.balanceVerifier.verifyBalanceMove({ + legacyUsdWalletId: migration.legacyUsdWalletId, + }), + ).rejects.toThrow("Not Found") + expect(getRawAccountDetails).toHaveBeenCalledTimes(1) + }) +}) From e8c8c18d716fb4e765e77d91db5173368128b303 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 16:03:03 -0400 Subject: [PATCH 3/7] feat(cutover): absorb sub-minimum dust fees + retry-failed operator command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two operator gaps from the ENG-461 TEST rehearsal: 1. Dust fees: IBEX rejects payments below its minimum payable amount, so sub-cent fee reimbursements (88-515 micros observed) 400'd at the pay step and stranded 9 migrations in requires_operator_review after their money had already moved. Fees under one cent are now absorbed instead of reimbursed (invisible to users): the balance_move_verified handler skips straight to fee_reimbursed, recording the absorbed micros for audit (feeAmountUsdtMicros kept, no payment transaction id). 2. retry-failed: failures previously required manual mongo surgery to re-drive. New CLI command resets `failed` migrations to their last safe status. Safe by construction — the runner routes money-moving failures to requires_operator_review, so `failed` records have no ambiguous side effects. Resume is field-driven: pointer flipped → pointer_flipped (preserving previousDefaultWalletId for rollback correctness); otherwise → not_started with stale progress cleared. Defense in depth: anything with a recorded payment transaction is skipped. Supports --account-id and --dry-run. transitionMigration gains an optional `clear` ($unset) arg for the retry reset; state machine gains failed → not_started | pointer_flipped. Refs ENG-484 --- src/app/cash-wallet-cutover/handlers.ts | 6 +- src/app/cash-wallet-cutover/index.ts | 1 + src/app/cash-wallet-cutover/retry-failed.ts | 131 ++++++++++++++++++ src/app/cash-wallet-cutover/state-machine.ts | 7 +- src/app/cash-wallet-cutover/worker.ts | 15 +- src/scripts/cash-wallet-cutover.ts | 17 +++ src/services/mongoose/cash-wallet-cutover.ts | 10 +- .../cash-wallet-cutover/retry-failed.spec.ts | 124 +++++++++++++++++ .../app/cash-wallet-cutover/worker.spec.ts | 33 +++++ 9 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 src/app/cash-wallet-cutover/retry-failed.ts create mode 100644 test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts diff --git a/src/app/cash-wallet-cutover/handlers.ts b/src/app/cash-wallet-cutover/handlers.ts index 28cdfdc15..f9afef03a 100644 --- a/src/app/cash-wallet-cutover/handlers.ts +++ b/src/app/cash-wallet-cutover/handlers.ts @@ -3,6 +3,7 @@ import { createCashWalletMigrationBalanceMoveInvoice, createCashWalletMigrationFeeReimbursementInvoice, flipCashWalletMigrationDefaultPointer, + isSubMinimumFeeReimbursementAmount, markCashWalletMigrationBalanceMoveSent, markCashWalletMigrationFeeReimbursed, provisionCashWalletMigrationDestination, @@ -131,10 +132,13 @@ export const createCashWalletMigrationStepHandlers = ({ const feeAmountUsdtMicros = await services.feeService.readFeeAmountUsdtMicros(migration) if (feeAmountUsdtMicros instanceof Error) return feeAmountUsdtMicros - if (feeAmountUsdtMicros === "0") { + const subMinimum = isSubMinimumFeeReimbursementAmount(feeAmountUsdtMicros) + if (subMinimum instanceof Error) return subMinimum + if (subMinimum) { return skipCashWalletMigrationFeeReimbursement({ migration, migrationsRepo, + feeAmountUsdtMicros, }) } return createCashWalletMigrationFeeReimbursementInvoice({ diff --git a/src/app/cash-wallet-cutover/index.ts b/src/app/cash-wallet-cutover/index.ts index e7d59d606..24960d7a3 100644 --- a/src/app/cash-wallet-cutover/index.ts +++ b/src/app/cash-wallet-cutover/index.ts @@ -20,6 +20,7 @@ export * from "./handlers" export * from "./runtime-services" export * from "./rollback-worker" export * from "./rollback" +export * from "./retry-failed" export * from "./orchestrator" export * from "./lifecycle" export * from "./preview" diff --git a/src/app/cash-wallet-cutover/retry-failed.ts b/src/app/cash-wallet-cutover/retry-failed.ts new file mode 100644 index 000000000..a118a3226 --- /dev/null +++ b/src/app/cash-wallet-cutover/retry-failed.ts @@ -0,0 +1,131 @@ +import { CashWalletCutoverRepository } from "@services/mongoose" + +import { CashWalletMigrationFailedError } from "./errors" + +type CashWalletRetryRepository = ReturnType + +export type CashWalletRetryFailedReport = { + dryRun: boolean + eligible: number + retried: number + /** Migrations skipped because money moved — those are operator-review work. */ + skippedMoneyMoved: number + resumedAt: Partial> + migrationIds: string[] +} + +/** + * Re-drive `failed` migrations (ENG-484). Safe by construction: the runner + * routes failures in money-moving statuses to `requires_operator_review`, so + * anything sitting in `failed` has no ambiguous side effects. Resume point is + * field-driven, mirroring the forward pipeline's guards: + * + * - pointer flipped (`previousDefaultWalletId` set) → resume `pointer_flipped`, + * preserving the recorded pointer so a later rollback stays correct + * - otherwise → reset to `not_started`, clearing stale progress fields so the + * machine re-walks provisioning and balance read from scratch + * + * Migrations with a recorded payment transaction are never retried here — + * defense in depth should one ever land in `failed`. + */ +export const retryFailedCashWalletMigrations = async ({ + cutoverVersion, + runId, + accountId, + dryRun = false, + migrationsRepo = CashWalletCutoverRepository(), +}: { + cutoverVersion: number + runId: string + /** Single-account mode when set; all failed migrations when omitted. */ + accountId?: AccountId + dryRun?: boolean + migrationsRepo?: CashWalletRetryRepository +}): Promise => { + let candidates: CashWalletMigration[] + + if (accountId !== undefined) { + const migration = await migrationsRepo.findMigrationByAccountId({ + accountId, + cutoverVersion, + runId, + }) + if (migration instanceof Error) return migration + if (migration === null) { + return new CashWalletMigrationFailedError( + `No migration found for accountId=${accountId} runId=${runId}`, + ) + } + if (migration.status !== "failed") { + return new CashWalletMigrationFailedError( + `Migration for accountId=${accountId} is ${migration.status}, not failed`, + ) + } + candidates = [migration] + } else { + const migrations = await migrationsRepo.listMigrationsByStatuses({ + cutoverVersion, + runId, + statuses: ["failed"], + }) + if (migrations instanceof Error) return migrations + candidates = migrations + } + + const report: CashWalletRetryFailedReport = { + dryRun, + eligible: 0, + retried: 0, + skippedMoneyMoved: 0, + resumedAt: {}, + migrationIds: [], + } + + for (const migration of candidates) { + if ( + migration.balanceMovePaymentTransactionId !== undefined || + migration.feeReimbursementPaymentTransactionId !== undefined + ) { + report.skippedMoneyMoved += 1 + continue + } + + const resumeStatus: CashWalletMigrationStatus = + migration.previousDefaultWalletId !== undefined + ? "pointer_flipped" + : "not_started" + + report.eligible += 1 + report.migrationIds.push(migration.id) + report.resumedAt[resumeStatus] = (report.resumedAt[resumeStatus] ?? 0) + 1 + if (dryRun) continue + + const transitioned = await migrationsRepo.transitionMigration({ + id: migration.id, + from: "failed", + to: resumeStatus, + cutoverVersion, + runId, + patch: { attempts: 0 }, + clear: + resumeStatus === "not_started" + ? [ + "lastError", + "sourceBalanceUsdCents", + "destinationAmountUsdtMicros", + "destinationStartingBalanceUsdtMicros", + "feeAmountUsdCents", + "feeAmountUsdtMicros", + "balanceMoveInvoicePaymentRequest", + "balanceMoveInvoicePaymentHash", + "feeReimbursementInvoicePaymentRequest", + "feeReimbursementInvoicePaymentHash", + ] + : ["lastError"], + }) + if (transitioned instanceof Error) return transitioned + report.retried += 1 + } + + return report +} diff --git a/src/app/cash-wallet-cutover/state-machine.ts b/src/app/cash-wallet-cutover/state-machine.ts index b6ff9158e..0325dce67 100644 --- a/src/app/cash-wallet-cutover/state-machine.ts +++ b/src/app/cash-wallet-cutover/state-machine.ts @@ -75,7 +75,12 @@ const transitions: Partial< pointer_flipped: ["legacy_zero_verified", "failed", "rollback_started"], legacy_zero_verified: ["complete", "failed", "rollback_started"], complete: ["rollback_started"], - failed: ["rollback_started"], + // Operator retry (ENG-484): `failed` only ever comes from non-ambiguous + // statuses (the runner routes ambiguous ones to requires_operator_review), + // so re-driving is safe. Two resume points: `not_started` re-walks the whole + // pre-money path; `pointer_flipped` resumes a post-pointer failure without + // touching previousDefaultWalletId. + failed: ["rollback_started", "not_started", "pointer_flipped"], requires_operator_review: ["rollback_started"], // Self-loop persists sub-step artifacts (invoice/payment ids) mid-rollback, // mirroring invoice_created's refresh self-loop. Rollback failures always diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index 2e48e0765..f4c4d80e5 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -91,6 +91,15 @@ const usesNoAmountFeeReimbursementInvoice = ( return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS } +// ENG-484: IBEX rejects payments below its minimum payable amount, so +// sub-cent fee reimbursements (88–515 micros observed in the ENG-461 +// rehearsal) 400 at the pay step and strand the migration. Fees under one +// cent are absorbed instead of reimbursed — invisible to users. +export const isSubMinimumFeeReimbursementAmount = ( + feeAmountUsdtMicros: string, +): boolean | InvalidCashWalletCutoverAmountError => + usesNoAmountFeeReimbursementInvoice(feeAmountUsdtMicros) + export const isInvoicePaymentRequestStale = ({ paymentRequest, now, @@ -417,9 +426,11 @@ export const createCashWalletMigrationFeeReimbursementInvoice = async ({ export const skipCashWalletMigrationFeeReimbursement = async ({ migration, migrationsRepo, + feeAmountUsdtMicros = "0", }: { migration: CashWalletMigration migrationsRepo: CashWalletMigrationTransitionRepository + feeAmountUsdtMicros?: string }): Promise => { const transition = assertCanTransition(migration.status, "fee_reimbursed") if (transition instanceof Error) return transition @@ -431,8 +442,10 @@ export const skipCashWalletMigrationFeeReimbursement = async ({ cutoverVersion: migration.cutoverVersion, runId: migration.runId, patch: { + // A skipped fee is always sub-cent, so cents are 0; the micros record + // what was absorbed (no feeReimbursementPaymentTransactionId = unpaid). feeAmountUsdCents: "0", - feeAmountUsdtMicros: "0", + feeAmountUsdtMicros, }, }) } diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 9c1806903..5ae99ac48 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -22,6 +22,10 @@ const args = yargs(hideBin(process.argv)) .command("prepare", "discover accounts and upsert migration records") .command("start", "mark a prepared cutover run in progress") .command("run-batch", "run one locked migration worker batch") + .command( + "retry-failed", + "reset failed migrations to their last safe status for re-running (single account with --account-id, else all failed)", + ) .command("status", "print cutover config and migration counts") .command("complete", "mark cutover complete after all migrations finish") .command( @@ -163,6 +167,19 @@ const run = async () => { return } + case "retry-failed": { + const result = await CashWalletCutover.retryFailedCashWalletMigrations({ + cutoverVersion, + runId, + accountId: args["account-id"] as AccountId | undefined, + dryRun: args["dry-run"], + migrationsRepo: repository, + }) + if (result instanceof Error) throw result + toJson(result) + return + } + case "rollback-request": { if (!args.reason) { throw new Error("--reason is required for rollback-request") diff --git a/src/services/mongoose/cash-wallet-cutover.ts b/src/services/mongoose/cash-wallet-cutover.ts index d0ce9f4c7..92297524a 100644 --- a/src/services/mongoose/cash-wallet-cutover.ts +++ b/src/services/mongoose/cash-wallet-cutover.ts @@ -34,6 +34,8 @@ type TransitionMigrationArgs = { cutoverVersion: number runId: string patch?: Partial> + /** Fields to $unset — e.g. clearing stale progress on a retry reset (ENG-484). */ + clear?: (keyof Omit)[] } type LockMigrationArgs = { @@ -208,11 +210,17 @@ export const CashWalletCutoverRepository = () => { cutoverVersion, runId, patch = {}, + clear = [], }: TransitionMigrationArgs): Promise => { try { const result = await CashWalletMigration.findOneAndUpdate( { _id: id, status: from, cutoverVersion, runId }, - { $set: { ...patch, status: to, updatedAt: new Date() } }, + { + $set: { ...patch, status: to, updatedAt: new Date() }, + ...(clear.length > 0 + ? { $unset: Object.fromEntries(clear.map((field) => [field, ""])) } + : {}), + }, { new: true }, ) if (!result) diff --git a/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts new file mode 100644 index 000000000..1b2b52499 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts @@ -0,0 +1,124 @@ +import { retryFailedCashWalletMigrations } from "@app/cash-wallet-cutover/retry-failed" + +const baseMigration = { + id: "migration-id", + accountId: "account-id" as AccountId, + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, + cutoverVersion: 1, + runId: "run-id", + status: "failed", + idempotencyKey: "run-id:account-id", + attempts: 3, + lastError: "FetchError: Too Many Requests", + updatedAt: new Date("2026-07-05T00:00:00.000Z"), +} as CashWalletMigration + +const repo = (migrations: CashWalletMigration[]) => ({ + findMigrationByAccountId: jest.fn(async ({ accountId }) => { + return migrations.find((m) => m.accountId === accountId) ?? null + }), + listMigrationsByStatuses: jest.fn(async () => migrations), + transitionMigration: jest.fn(async ({ to, patch }) => ({ + ...migrations[0], + ...patch, + status: to, + })), +}) + +describe("retryFailedCashWalletMigrations", () => { + it("resets a pre-money failure to not_started, clearing stale progress", async () => { + const migrationsRepo = repo([ + { ...baseMigration, sourceBalanceUsdCents: "100" } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ + eligible: 1, + retried: 1, + skippedMoneyMoved: 0, + resumedAt: { not_started: 1 }, + }) + expect(migrationsRepo.transitionMigration).toHaveBeenCalledWith( + expect.objectContaining({ + from: "failed", + to: "not_started", + patch: { attempts: 0 }, + clear: expect.arrayContaining(["lastError", "sourceBalanceUsdCents"]), + }), + ) + }) + + it("resumes a post-pointer failure at pointer_flipped, preserving the pointer", async () => { + const migrationsRepo = repo([ + { + ...baseMigration, + previousDefaultWalletId: "legacy-wallet-id" as WalletId, + } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ retried: 1, resumedAt: { pointer_flipped: 1 } }) + const call = migrationsRepo.transitionMigration.mock.calls[0][0] + expect(call.to).toBe("pointer_flipped") + expect(call.clear).toEqual(["lastError"]) + expect(call.clear).not.toContain("previousDefaultWalletId") + }) + + it("never retries a failure with a recorded payment transaction", async () => { + const migrationsRepo = repo([ + { + ...baseMigration, + balanceMovePaymentTransactionId: "txn-id", + } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ eligible: 0, retried: 0, skippedMoneyMoved: 1 }) + expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() + }) + + it("dry-run reports without writing", async () => { + const migrationsRepo = repo([baseMigration]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + dryRun: true, + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ dryRun: true, eligible: 1, retried: 0 }) + expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() + }) + + it("rejects single-account retry when the migration is not failed", async () => { + const migrationsRepo = repo([ + { ...baseMigration, status: "complete" } as CashWalletMigration, + ]) + + const result = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + accountId: "account-id" as AccountId, + migrationsRepo: migrationsRepo as never, + }) + + expect(result).toBeInstanceOf(Error) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts index 32ffd382e..ce4dbced6 100644 --- a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts @@ -1,6 +1,8 @@ import { createCashWalletMigrationFeeReimbursementInvoice, + isSubMinimumFeeReimbursementAmount, sendCashWalletMigrationFeeReimbursementPayment, + skipCashWalletMigrationFeeReimbursement, } from "@app/cash-wallet-cutover/worker" const migration = { @@ -90,4 +92,35 @@ describe("cash wallet migration worker fee reimbursement", () => { senderAmountUsdtMicros: "4735", }) }) + + it("classifies sub-cent fees as sub-minimum (absorbed, never paid)", () => { + expect(isSubMinimumFeeReimbursementAmount("0")).toBe(true) + expect(isSubMinimumFeeReimbursementAmount("515")).toBe(true) // rehearsal dust + expect(isSubMinimumFeeReimbursementAmount("9999")).toBe(true) + expect(isSubMinimumFeeReimbursementAmount("10000")).toBe(false) + expect(isSubMinimumFeeReimbursementAmount("-1")).toBeInstanceOf(Error) + }) + + it("records the absorbed micros when skipping a dust fee (ENG-484)", async () => { + const migrationsRepo = transitionRepo() + + const result = await skipCashWalletMigrationFeeReimbursement({ + migration, + migrationsRepo, + feeAmountUsdtMicros: "515", + }) + + expect(result).toEqual({ + ...migration, + status: "fee_reimbursed", + feeAmountUsdCents: "0", + feeAmountUsdtMicros: "515", + }) + expect(migrationsRepo.transitionMigration).toHaveBeenCalledWith( + expect.objectContaining({ + to: "fee_reimbursed", + patch: { feeAmountUsdCents: "0", feeAmountUsdtMicros: "515" }, + }), + ) + }) }) From 70537ef364eac77fd2c021c1b577b9f3fab2eebf Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 16:36:38 -0400 Subject: [PATCH 4/7] style: prettier fix in retry-failed Co-Authored-By: Claude Opus 4.8 --- src/app/cash-wallet-cutover/retry-failed.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app/cash-wallet-cutover/retry-failed.ts b/src/app/cash-wallet-cutover/retry-failed.ts index a118a3226..9b8457aec 100644 --- a/src/app/cash-wallet-cutover/retry-failed.ts +++ b/src/app/cash-wallet-cutover/retry-failed.ts @@ -91,9 +91,7 @@ export const retryFailedCashWalletMigrations = async ({ } const resumeStatus: CashWalletMigrationStatus = - migration.previousDefaultWalletId !== undefined - ? "pointer_flipped" - : "not_started" + migration.previousDefaultWalletId !== undefined ? "pointer_flipped" : "not_started" report.eligible += 1 report.migrationIds.push(migration.id) From 0926cc46f3e90fede8ab300080ef67ce89822605 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 16:38:45 -0400 Subject: [PATCH 5/7] feat(cutover): resolve treasury to the funder's USDT wallet (ENG-482) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getTreasuryWalletId returned the funder's DEFAULT wallet. On envs where that default is BTC, USDT fee reimbursements (and rollback top-ups) 404 at IBEX — which stranded every funded account in the ENG-461 rehearsal until the TEST funder's default was manually flipped to USDT. The cutover treasury resolver now finds the funder's USDT wallet regardless of which wallet is the default (mirroring the dealer resolver's find-by-currency pattern), so prod does not depend on the default-flip operational step. If the funder has no USDT wallet at all, it returns a descriptive error naming the missing prerequisite instead of a bare IBEX 404 at payment time. The global funderWalletResolver is unchanged — other funder flows keep their existing default-wallet semantics. Refs ENG-482 --- .../cash-wallet-cutover/runtime-services.ts | 33 +++++++++- .../runtime-services.spec.ts | 66 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index 2b33c78f2..5adf4cd1f 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -3,7 +3,7 @@ import { decodeInvoice } from "@domain/bitcoin/lightning" import { InvalidWalletId } from "@domain/errors" import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" import { WalletType } from "@domain/wallets" -import { AccountsRepository } from "@services/mongoose" +import { AccountsRepository, WalletsRepository } from "@services/mongoose" import Ibex from "@services/ibex/client" import { UnexpectedIbexResponse } from "@services/ibex/errors" import { getFunderWalletId } from "@services/ledger/caching" @@ -30,6 +30,8 @@ type RuntimeServiceDependencies = { createNoAmountInvoice?: typeof Ibex.addInvoice payInvoice?: typeof Ibex.payInvoice accountsRepo?: Pick, "findById"> + walletsRepo?: Pick, "findById" | "listByAccountId"> + getFunderWalletId?: typeof getFunderWalletId getTreasuryWalletId?: () => Promise maxRateLimitAttempts?: number rateLimitRetryDelayMs?: number @@ -174,6 +176,8 @@ export const createCashWalletMigrationRuntimeServices = ( const noAmountInvoiceForRecipient = deps.createNoAmountInvoice ?? Ibex.addInvoice const payInvoice = deps.payInvoice ?? Ibex.payInvoice const accountsRepo = deps.accountsRepo ?? AccountsRepository() + const walletsRepo = deps.walletsRepo ?? WalletsRepository() + const funderWalletId = deps.getFunderWalletId ?? getFunderWalletId const rateLimitRetry = { maxAttempts: Math.max( 1, @@ -398,7 +402,32 @@ export const createCashWalletMigrationRuntimeServices = ( }, }, treasuryService: { - getTreasuryWalletId: deps.getTreasuryWalletId ?? getFunderWalletId, + // ENG-482: the treasury must be a USDT wallet — fee reimbursements and + // rollback top-ups pay USDT invoices, and paying them from the funder's + // BTC default 404s at IBEX (stranded every funded account in the ENG-461 + // rehearsal). Resolve the funder's USDT wallet regardless of which + // wallet is its default, so prod doesn't depend on flipping the funder + // default as an operational step. + getTreasuryWalletId: + deps.getTreasuryWalletId ?? + (async (): Promise => { + const funderDefaultWalletId = await funderWalletId() + const funderWallet = await walletsRepo.findById(funderDefaultWalletId) + if (funderWallet instanceof Error) return funderWallet + if (funderWallet.currency === WalletCurrency.Usdt) return funderWallet.id + + const funderWallets = await walletsRepo.listByAccountId(funderWallet.accountId) + if (funderWallets instanceof Error) return funderWallets + const usdtWallet = funderWallets.find( + (wallet) => wallet.currency === WalletCurrency.Usdt, + ) + if (usdtWallet === undefined) { + return new CashWalletMigrationFailedError( + "Funder account has no USDT wallet — create and fund the cutover treasury before running the cutover (ENG-482)", + ) + } + return usdtWallet.id + }), }, pointerService: { flipDefaultWallet: async ({ diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index 731b50b69..43edb258c 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -66,6 +66,72 @@ describe("cutover runtime services — IBEX rate-limit retry (ENG-483)", () => { expect(getRawAccountDetails).toHaveBeenCalledTimes(3) }) + it("resolves the treasury to the funder's USDT wallet regardless of its default (ENG-482)", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest.fn().mockResolvedValue([ + { id: "funder-btc-wallet", currency: "BTC" }, + { id: "funder-usdt-wallet", currency: "USDT" }, + ]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + await expect(services.treasuryService.getTreasuryWalletId()).resolves.toBe( + "funder-usdt-wallet", + ) + }) + + it("short-circuits when the funder default is already the USDT wallet", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-usdt-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "USDT", + }), + listByAccountId: jest.fn(), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-usdt-wallet" as WalletId, + }) + + await expect(services.treasuryService.getTreasuryWalletId()).resolves.toBe( + "funder-usdt-wallet", + ) + expect(walletsRepo.listByAccountId).not.toHaveBeenCalled() + }) + + it("returns a descriptive error when the funder has no USDT wallet", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest + .fn() + .mockResolvedValue([{ id: "funder-btc-wallet", currency: "BTC" }]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + const result = await services.treasuryService.getTreasuryWalletId() + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/no USDT wallet/) + }) + it("does not retry non-rate-limit rejections", async () => { const getRawAccountDetails = jest .fn() From bc413f4eaa2233e86515962de0cf7ba5810fc77d Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 16:44:26 -0400 Subject: [PATCH 6/7] =?UTF-8?q?feat(cutover):=20audit-accounts=20command?= =?UTF-8?q?=20=E2=80=94=20find=20accounts=20that=20break=20mid-migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pointer-flip step saves the whole account document, so mongoose re-validates every field: accounts with pre-existing invalid data fail mid-migration even though the cutover never touched those fields. The ENG-461 rehearsal hit two live: an empty username (min length 3) failed an account at pointer flip, and a null statusHistory status blocked the funder default-wallet update. New `audit-accounts` CLI command sweeps every account document through validateSync() and reports the ones that would fail, with per-field messages. Runbook D1 prerequisite: run before `prepare`; every issue is an account to fix or lock before the run. Refs ENG-484 --- src/app/cash-wallet-cutover/audit-accounts.ts | 58 ++++++++++++++++ src/app/cash-wallet-cutover/index.ts | 1 + src/scripts/cash-wallet-cutover.ts | 10 +++ .../audit-accounts.spec.ts | 69 +++++++++++++++++++ 4 files changed, 138 insertions(+) create mode 100644 src/app/cash-wallet-cutover/audit-accounts.ts create mode 100644 test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts diff --git a/src/app/cash-wallet-cutover/audit-accounts.ts b/src/app/cash-wallet-cutover/audit-accounts.ts new file mode 100644 index 000000000..2741ef800 --- /dev/null +++ b/src/app/cash-wallet-cutover/audit-accounts.ts @@ -0,0 +1,58 @@ +import { Account } from "@services/mongoose/schema" + +export type CashWalletCutoverAccountAuditIssue = { + accountId: string + role?: string + errors: string[] +} + +export type CashWalletCutoverAccountAuditReport = { + scanned: number + invalid: number + issues: CashWalletCutoverAccountAuditIssue[] +} + +type ValidatableAccountDocument = { + _id: { toString(): string } + role?: string + validateSync(): { errors: Record } | undefined +} + +/** + * Data-quality audit (ENG-484 / cutover runbook D1). The pointer-flip step + * saves the whole account document, so mongoose re-validates EVERY field — + * an account with pre-existing invalid data (empty username, null + * statusHistory entry, ...) fails mid-migration even though the cutover never + * touched those fields. Both were hit live in the ENG-461 rehearsal. + * + * Run before `prepare`: every issue reported here is an account that WILL + * fail at pointer flip unless fixed (or locked out of the run). + */ +export const auditCashWalletCutoverAccounts = async ({ + listAccountDocuments = () => + Account.find({}).cursor() as unknown as AsyncIterable, +}: { + listAccountDocuments?: () => AsyncIterable +} = {}): Promise => { + const report: CashWalletCutoverAccountAuditReport = { + scanned: 0, + invalid: 0, + issues: [], + } + + for await (const document of listAccountDocuments()) { + report.scanned += 1 + + const validation = document.validateSync() + if (validation === undefined) continue + + report.invalid += 1 + report.issues.push({ + accountId: document._id.toString(), + role: document.role, + errors: Object.values(validation.errors).map(({ message }) => message), + }) + } + + return report +} diff --git a/src/app/cash-wallet-cutover/index.ts b/src/app/cash-wallet-cutover/index.ts index 24960d7a3..02334252c 100644 --- a/src/app/cash-wallet-cutover/index.ts +++ b/src/app/cash-wallet-cutover/index.ts @@ -21,6 +21,7 @@ export * from "./runtime-services" export * from "./rollback-worker" export * from "./rollback" export * from "./retry-failed" +export * from "./audit-accounts" export * from "./orchestrator" export * from "./lifecycle" export * from "./preview" diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 5ae99ac48..0a45500f7 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -15,6 +15,10 @@ import { baseLogger } from "@services/logger" const args = yargs(hideBin(process.argv)) .command("preview", "discover accounts and print the migration plan without writes") + .command( + "audit-accounts", + "validate every account document; reports accounts that would fail the pointer-flip save (run before prepare)", + ) .command( "provision-usdt-wallets", "create missing destination USDT wallets before preparing migrations", @@ -80,6 +84,12 @@ const run = async () => { return } + case "audit-accounts": { + const result = await CashWalletCutover.auditCashWalletCutoverAccounts() + toJson(result) + return + } + case "provision-usdt-wallets": { const result = await CashWalletCutover.provisionPrimaryCashWalletUsdtWallets({ cutoverVersion, diff --git a/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts b/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts new file mode 100644 index 000000000..0c81d877d --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts @@ -0,0 +1,69 @@ +import { auditCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/audit-accounts" + +const doc = ({ + id, + role, + errors, +}: { + id: string + role?: string + errors?: Record +}) => ({ + _id: { toString: () => id }, + role, + validateSync: () => (errors ? { errors } : undefined), +}) + +async function* documents(...docs: ReturnType[]) { + yield* docs +} + +describe("auditCashWalletCutoverAccounts", () => { + it("reports accounts whose document fails mongoose validation", async () => { + const report = await auditCashWalletCutoverAccounts({ + listAccountDocuments: () => + documents( + doc({ id: "good-account" }), + doc({ + id: "empty-username", + errors: { + username: { + message: + "Path `username` (``) is shorter than the minimum allowed length (3).", + }, + }, + }), + doc({ + id: "null-status", + role: "funder", + errors: { + "statusHistory.0.status": { message: "Path `status` is required." }, + }, + }), + ), + }) + + expect(report.scanned).toBe(3) + expect(report.invalid).toBe(2) + expect(report.issues).toEqual([ + { + accountId: "empty-username", + role: undefined, + errors: [expect.stringMatching(/username/)], + }, + { + accountId: "null-status", + role: "funder", + errors: ["Path `status` is required."], + }, + ]) + }) + + it("reports a clean sweep when every account validates", async () => { + const report = await auditCashWalletCutoverAccounts({ + listAccountDocuments: () => documents(doc({ id: "a" }), doc({ id: "b" })), + }) + + expect(report).toEqual({ scanned: 2, invalid: 0, issues: [] }) + }) +}) From b0ac041794a8adb5c5326b5696b476cae6bfa277 Mon Sep 17 00:00:00 2001 From: Dread Date: Sun, 5 Jul 2026 19:00:46 -0400 Subject: [PATCH 7/7] fix(cutover): address PR #433 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fee absorption decoupled from the fixed-invoice constant: new CUTOVER_MIN_PAYABLE_USDT_MICROS = 2500 (empirically measured on TEST: 600 micros -> IBEX 400, 650+ pays — IBEX's floor is the LN 1-sat minimum, which floats with BTC price; 2500 covers it to ~$250k/BTC and stays under a cent). The 2500-9999 band reimburses again via the no-amount invoice path, which is live code once more. - Rollback: fail closed with a clear message when the reverse amount is below the minimum payable (drained/1-cent wallets previously produced a doomed pay attempt and a misleading IBEX 400 in review). - Rollback: documented the accepted step-3 boundary tradeoff where the treasury can reimburse up to ~1 cent of post-cutover user spend. - State machine: corrected the retry-edge comment — failed CAN hold money-moved migrations (legacy_zero_verified failures); the edges are safe behind retry-failed's guards, now stated explicitly. - retry-failed: a lost race (migration concurrently moved out of failed) no longer aborts the run and discards the report — it's skipped and surfaced via a new conflicts counter. - Treasury resolver: memoized for the life of the run (was per-migration mongo reads of an invariant id) and fails closed on duplicate funder USDT wallets instead of paying from an arbitrary one. - Reconciliation audit: absorbed (recorded-but-unpaid) fees no longer render as false shortfall rows — the allowance is exactly the recorded fee, and only when no reimbursement transaction exists. 12 suites / 68 tests green (5 new). --- .../cash-wallet-cutover/operator-dashboard.ts | 13 +++- src/app/cash-wallet-cutover/retry-failed.ts | 15 ++++- .../cash-wallet-cutover/rollback-worker.ts | 19 +++++- .../cash-wallet-cutover/runtime-services.ts | 53 ++++++++++++----- src/app/cash-wallet-cutover/state-machine.ts | 14 +++-- src/app/cash-wallet-cutover/worker.ts | 25 ++++++-- .../cash-wallet-cutover/balance-audit.spec.ts | 59 +++++++++++++++++++ .../cash-wallet-cutover/retry-failed.spec.ts | 26 ++++++++ .../rollback-worker.spec.ts | 23 ++++++++ .../runtime-services.spec.ts | 45 ++++++++++++++ .../app/cash-wallet-cutover/worker.spec.ts | 12 ++-- 11 files changed, 269 insertions(+), 35 deletions(-) create mode 100644 test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index 982c6ff64..b6832c0e8 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -491,7 +491,7 @@ const parseIntegerAmount = (value?: string): number | undefined => { return Number(value) } -const computeCutoverBalanceAudit = ({ +export const computeCutoverBalanceAudit = ({ migration, usdtWallets, }: { @@ -539,9 +539,18 @@ const computeCutoverBalanceAudit = ({ 0, currentDestinationBalanceUsdtMicros - destinationStartingBalanceUsdtMicros, ) + // ENG-484: an absorbed fee (recorded but never reimbursed — no payment + // transaction id) legitimately leaves the destination short of target by + // exactly that fee. Without this allowance every dust-fee account renders + // as a red "shortfall" row during the post-run reconciliation hold, + // drowning real shortfalls. + const absorbedFeeUsdtMicros = + migration.feeReimbursementPaymentTransactionId === undefined + ? (parseIntegerAmount(migration.feeAmountUsdtMicros) ?? 0) + : 0 const shortfallUsdtMicros = Math.max( 0, - expectedMinimumUsdtMicros - finalDeltaUsdtMicros, + expectedMinimumUsdtMicros - absorbedFeeUsdtMicros - finalDeltaUsdtMicros, ) const roundingSubsidyUsdtMicros = Math.max( 0, diff --git a/src/app/cash-wallet-cutover/retry-failed.ts b/src/app/cash-wallet-cutover/retry-failed.ts index 9b8457aec..8b2b5f4e2 100644 --- a/src/app/cash-wallet-cutover/retry-failed.ts +++ b/src/app/cash-wallet-cutover/retry-failed.ts @@ -10,6 +10,8 @@ export type CashWalletRetryFailedReport = { retried: number /** Migrations skipped because money moved — those are operator-review work. */ skippedMoneyMoved: number + /** Lost races: migrations concurrently moved out of `failed` mid-run. */ + conflicts: number resumedAt: Partial> migrationIds: string[] } @@ -77,6 +79,7 @@ export const retryFailedCashWalletMigrations = async ({ eligible: 0, retried: 0, skippedMoneyMoved: 0, + conflicts: 0, resumedAt: {}, migrationIds: [], } @@ -121,7 +124,17 @@ export const retryFailedCashWalletMigrations = async ({ ] : ["lastError"], }) - if (transitioned instanceof Error) return transitioned + if (transitioned instanceof Error) { + // Lost a race (e.g. a concurrent rollback-request moved this migration + // out of `failed` — the guarded findOneAndUpdate matched nothing). + // Skip it and keep going: aborting would discard the report of + // migrations already reset by earlier iterations. + report.eligible -= 1 + report.migrationIds.pop() + report.resumedAt[resumeStatus] = (report.resumedAt[resumeStatus] ?? 1) - 1 + report.conflicts += 1 + continue + } report.retried += 1 } diff --git a/src/app/cash-wallet-cutover/rollback-worker.ts b/src/app/cash-wallet-cutover/rollback-worker.ts index 310a64070..4e3bade03 100644 --- a/src/app/cash-wallet-cutover/rollback-worker.ts +++ b/src/app/cash-wallet-cutover/rollback-worker.ts @@ -1,6 +1,6 @@ import { assertCanTransition, ROLLBACKABLE_STATUSES } from "./state-machine" import { legacyShortfallUsdtMicros } from "./amount-conversion" -import { isInvoicePaymentRequestStale } from "./worker" +import { CUTOVER_MIN_PAYABLE_USDT_MICROS, isInvoicePaymentRequestStale } from "./worker" import { CashWalletMigrationFailedError, InvalidCashWalletMigrationTransitionError, @@ -218,6 +218,15 @@ export const executeCashWalletMigrationRollbackStep = async ({ `USDT balance ${spendableStr} is short of reverse target ${target} by ${shortfall} micros (> tolerance) — likely spent post-cutover`, ) } + // IBEX rejects payments below ~1 sat (ENG-484): a drained or 1-cent + // wallet can pass the tolerance gate with a 0- or sub-minimum reverse + // amount — fail closed with a clear message instead of a doomed pay + // attempt surfacing as a misleading IBEX 400. + if (reverseAmount < CUTOVER_MIN_PAYABLE_USDT_MICROS) { + return new CashWalletMigrationFailedError( + `Reverse amount ${reverseAmount} micros is below IBEX's minimum payable — target ${target}, spendable ${spendableStr}; resolve manually`, + ) + } const reverseAmountStr = reverseAmount.toString() if ( @@ -287,6 +296,14 @@ export const executeCashWalletMigrationRollbackStep = async ({ }) if (shortfall instanceof Error) return shortfall + // Accepted boundary tradeoff: step 2 tolerates up to a 1-cent USDT + // shortfall (user spend + un-reimbursed fee), and this top-up covers + // whatever the legacy wallet is short vs the original balance — so up to + // ~1 cent of a user's own post-cutover spending can be reimbursed by the + // treasury (spend ≤ tolerance at step 2, but spend + fee + receive dust + // over it here). Exposure is bounded per account at roughly a cent plus + // fee plus dust; the old strict fail-closed left every such account in + // manual review instead. Documented in ENG-484. if (BigInt(shortfall) > ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS) { if (current.rollbackShortfallPaymentTransactionId !== undefined) { // Treasury already topped up once and the user is still short more diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index 5adf4cd1f..ce49a7028 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -408,26 +408,47 @@ export const createCashWalletMigrationRuntimeServices = ( // rehearsal). Resolve the funder's USDT wallet regardless of which // wallet is its default, so prod doesn't depend on flipping the funder // default as an operational step. + // Memoized: the id is invariant for the life of the run and this is + // called per migration at the fee step — without the cache a transient + // mongo blip mid-run fails a step the old memoized getFunderWalletId + // never would have. getTreasuryWalletId: deps.getTreasuryWalletId ?? - (async (): Promise => { - const funderDefaultWalletId = await funderWalletId() - const funderWallet = await walletsRepo.findById(funderDefaultWalletId) - if (funderWallet instanceof Error) return funderWallet - if (funderWallet.currency === WalletCurrency.Usdt) return funderWallet.id - - const funderWallets = await walletsRepo.listByAccountId(funderWallet.accountId) - if (funderWallets instanceof Error) return funderWallets - const usdtWallet = funderWallets.find( - (wallet) => wallet.currency === WalletCurrency.Usdt, - ) - if (usdtWallet === undefined) { - return new CashWalletMigrationFailedError( - "Funder account has no USDT wallet — create and fund the cutover treasury before running the cutover (ENG-482)", + (() => { + let cached: WalletId | undefined + return async (): Promise => { + if (cached !== undefined) return cached + const funderDefaultWalletId = await funderWalletId() + const funderWallet = await walletsRepo.findById(funderDefaultWalletId) + if (funderWallet instanceof Error) return funderWallet + if (funderWallet.currency === WalletCurrency.Usdt) { + cached = funderWallet.id + return cached + } + + const funderWallets = await walletsRepo.listByAccountId( + funderWallet.accountId, + ) + if (funderWallets instanceof Error) return funderWallets + const usdtWallets = funderWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usdt, ) + if (usdtWallets.length === 0) { + return new CashWalletMigrationFailedError( + "Funder account has no USDT wallet — create and fund the cutover treasury before running the cutover (ENG-482)", + ) + } + if (usdtWallets.length > 1) { + // No unique index enforces one-USDT-wallet-per-account; picking + // one blind risks paying from an unfunded duplicate. + return new CashWalletMigrationFailedError( + `Funder account has ${usdtWallets.length} USDT wallets — resolve the duplicate before running the cutover (ENG-482)`, + ) + } + cached = usdtWallets[0].id + return cached } - return usdtWallet.id - }), + })(), }, pointerService: { flipDefaultWallet: async ({ diff --git a/src/app/cash-wallet-cutover/state-machine.ts b/src/app/cash-wallet-cutover/state-machine.ts index 0325dce67..02dc6714f 100644 --- a/src/app/cash-wallet-cutover/state-machine.ts +++ b/src/app/cash-wallet-cutover/state-machine.ts @@ -75,11 +75,15 @@ const transitions: Partial< pointer_flipped: ["legacy_zero_verified", "failed", "rollback_started"], legacy_zero_verified: ["complete", "failed", "rollback_started"], complete: ["rollback_started"], - // Operator retry (ENG-484): `failed` only ever comes from non-ambiguous - // statuses (the runner routes ambiguous ones to requires_operator_review), - // so re-driving is safe. Two resume points: `not_started` re-walks the whole - // pre-money path; `pointer_flipped` resumes a post-pointer failure without - // touching previousDefaultWalletId. + // Operator retry (ENG-484). NOTE: `failed` CAN hold money-moved migrations — + // a failure during legacy_zero_verified (balance moved AND pointer flipped) + // routes to `failed` because that status is not in the runner's + // AMBIGUOUS_SIDE_EFFECT_STATUSES (kept that way deliberately: those + // failures are usually transient verify reads and stay auto-retryable). + // These edges are only safe behind retry-failed's guards: skip any + // migration with a recorded payment transaction id, and resume at + // `pointer_flipped` (never `not_started`) when previousDefaultWalletId + // is set. failed: ["rollback_started", "not_started", "pointer_flipped"], requires_operator_review: ["rollback_started"], // Self-loop persists sub-step artifacts (invoice/payment ids) mid-rollback, diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index f4c4d80e5..3cfd4bcc6 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -91,14 +91,27 @@ const usesNoAmountFeeReimbursementInvoice = ( return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS } -// ENG-484: IBEX rejects payments below its minimum payable amount, so -// sub-cent fee reimbursements (88–515 micros observed in the ENG-461 -// rehearsal) 400 at the pay step and strand the migration. Fees under one -// cent are absorbed instead of reimbursed — invisible to users. +// ENG-484: IBEX rejects Lightning payments below ~1 satoshi. Measured on TEST +// (2026-07-05, USDT→USDT no-amount invoices): 600 micros → 400 Bad Request, +// 650+ → paid. A 1-sat floor FLOATS with BTC price, so this margin covers it +// up to ~$250k/BTC. Fees below this are absorbed instead of reimbursed +// (≤ ¼ cent — invisible to users); fees above it still pay via the no-amount +// invoice path (which handles 1–9999 micros, e.g. the 4735-micro case below). +// Deliberately NOT tied to CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS: that is an +// invoice-format constant, this is a payability floor + customer-funds policy. +export const CUTOVER_MIN_PAYABLE_USDT_MICROS = 2_500n + export const isSubMinimumFeeReimbursementAmount = ( feeAmountUsdtMicros: string, -): boolean | InvalidCashWalletCutoverAmountError => - usesNoAmountFeeReimbursementInvoice(feeAmountUsdtMicros) +): boolean | InvalidCashWalletCutoverAmountError => { + if (!/^\d+$/.test(feeAmountUsdtMicros)) { + return new InvalidCashWalletCutoverAmountError( + `Invalid non-negative integer amount: ${feeAmountUsdtMicros}`, + ) + } + + return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_PAYABLE_USDT_MICROS +} export const isInvoicePaymentRequestStale = ({ paymentRequest, diff --git a/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts b/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts new file mode 100644 index 000000000..bdbb08fc7 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts @@ -0,0 +1,59 @@ +import { computeCutoverBalanceAudit } from "@app/cash-wallet-cutover/operator-dashboard" + +const migration = ({ + feePaid, + feeAmountUsdtMicros = "515", +}: { + feePaid: boolean + feeAmountUsdtMicros?: string +}) => + ({ + id: "migration-id", + status: "complete", + sourceBalanceUsdCents: "100", + destinationAmountUsdtMicros: "10000", + destinationStartingBalanceUsdtMicros: "0", + destinationUsdtWalletId: "dest-wallet" as WalletId, + feeAmountUsdtMicros, + ...(feePaid ? { feeReimbursementPaymentTransactionId: "fee-txn" } : {}), + }) as CashWalletMigration + +const wallet = (minorUnitsNumber: number) => + ({ + id: "dest-wallet", + balance: { status: "fresh", minorUnitsNumber }, + }) as never + +describe("computeCutoverBalanceAudit — absorbed-fee allowance (ENG-484)", () => { + it("does not flag an absorbed fee as a shortfall", () => { + // Fee recorded but never reimbursed: destination legitimately holds + // target − fee. Must render verified, not a red shortfall row. + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: false }), + usdtWallets: [wallet(9485)], // 10000 − 515 absorbed + }) + + expect(audit?.status).toBe("verified") + expect(audit?.shortfallUsdtMicros).toBe(0) + }) + + it("still flags a real shortfall beyond the absorbed fee", () => { + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: false }), + usdtWallets: [wallet(8000)], // short 1485 beyond the absorbed 515 + }) + + expect(audit?.status).toBe("shortfall") + expect(audit?.shortfallUsdtMicros).toBe(1485) + }) + + it("grants no allowance when the fee was actually reimbursed", () => { + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: true, feeAmountUsdtMicros: "5000" }), + usdtWallets: [wallet(9000)], + }) + + expect(audit?.status).toBe("shortfall") + expect(audit?.shortfallUsdtMicros).toBe(1000) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts index 1b2b52499..ee92628f3 100644 --- a/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts @@ -107,6 +107,32 @@ describe("retryFailedCashWalletMigrations", () => { expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() }) + it("skips a lost race and keeps the report of migrations already reset", async () => { + const second = { + ...baseMigration, + id: "migration-2", + accountId: "account-2" as AccountId, + } as CashWalletMigration + const migrationsRepo = repo([baseMigration, second]) + migrationsRepo.transitionMigration + .mockResolvedValueOnce({ ...baseMigration, status: "not_started" }) + .mockResolvedValueOnce(new Error("Could not transition cash wallet migration")) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ + retried: 1, + conflicts: 1, + eligible: 1, + migrationIds: ["migration-id"], + resumedAt: { not_started: 1 }, + }) + }) + it("rejects single-account retry when the migration is not failed", async () => { const migrationsRepo = repo([ { ...baseMigration, status: "complete" } as CashWalletMigration, diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts index 3d3775213..8e72584a3 100644 --- a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts @@ -258,6 +258,29 @@ describe("rollback executor", () => { expect(result).toMatchObject({ status: "rolled_back" }) }) + it("fails closed when the reverse amount is below IBEX's minimum payable (ENG-484)", async () => { + // A 1-cent account with a nearly-drained wallet passes the tolerance gate + // (shortfall ≤ 1 cent) but the reverse amount is unpayable — fail closed + // with a clear message instead of a doomed pay and a misleading IBEX 400. + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "1", + destinationAmountUsdtMicros: "10000", + } + const svc = services({ spendableUsdtMicros: "2000" }) // below 2500 min payable + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/below IBEX's minimum payable/) + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + }) + it("fails closed when the USDT balance no longer covers the reverse amount", async () => { const migration = { ...baseMigration, diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index 43edb258c..3dc469e49 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -110,6 +110,51 @@ describe("cutover runtime services — IBEX rate-limit retry (ENG-483)", () => { expect(walletsRepo.listByAccountId).not.toHaveBeenCalled() }) + it("memoizes treasury resolution — one lookup for the whole run", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-usdt-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "USDT", + }), + listByAccountId: jest.fn(), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-usdt-wallet" as WalletId, + }) + + await services.treasuryService.getTreasuryWalletId() + await services.treasuryService.getTreasuryWalletId() + await services.treasuryService.getTreasuryWalletId() + + expect(walletsRepo.findById).toHaveBeenCalledTimes(1) + }) + + it("fails closed when the funder has duplicate USDT wallets", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest.fn().mockResolvedValue([ + { id: "usdt-1", currency: "USDT" }, + { id: "usdt-2", currency: "USDT" }, + ]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + const result = await services.treasuryService.getTreasuryWalletId() + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/2 USDT wallets/) + }) + it("returns a descriptive error when the funder has no USDT wallet", async () => { const walletsRepo = { findById: jest.fn().mockResolvedValue({ diff --git a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts index ce4dbced6..1396268d8 100644 --- a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts @@ -93,11 +93,15 @@ describe("cash wallet migration worker fee reimbursement", () => { }) }) - it("classifies sub-cent fees as sub-minimum (absorbed, never paid)", () => { + it("classifies only sub-minimum-payable fees as absorbed", () => { expect(isSubMinimumFeeReimbursementAmount("0")).toBe(true) - expect(isSubMinimumFeeReimbursementAmount("515")).toBe(true) // rehearsal dust - expect(isSubMinimumFeeReimbursementAmount("9999")).toBe(true) - expect(isSubMinimumFeeReimbursementAmount("10000")).toBe(false) + expect(isSubMinimumFeeReimbursementAmount("515")).toBe(true) // rehearsal dust (observed 400) + expect(isSubMinimumFeeReimbursementAmount("2499")).toBe(true) + expect(isSubMinimumFeeReimbursementAmount("2500")).toBe(false) + // payable via the no-amount invoice path — NOT absorbed (reviewer: the + // 2500–9999 band reimbursed successfully before ENG-484) + expect(isSubMinimumFeeReimbursementAmount("4735")).toBe(false) + expect(isSubMinimumFeeReimbursementAmount("9999")).toBe(false) expect(isSubMinimumFeeReimbursementAmount("-1")).toBeInstanceOf(Error) })