diff --git a/src/app/cash-wallet-cutover/audit-accounts.ts b/src/app/cash-wallet-cutover/audit-accounts.ts new file mode 100644 index 000000000..2741ef800 --- /dev/null +++ b/src/app/cash-wallet-cutover/audit-accounts.ts @@ -0,0 +1,58 @@ +import { Account } from "@services/mongoose/schema" + +export type CashWalletCutoverAccountAuditIssue = { + accountId: string + role?: string + errors: string[] +} + +export type CashWalletCutoverAccountAuditReport = { + scanned: number + invalid: number + issues: CashWalletCutoverAccountAuditIssue[] +} + +type ValidatableAccountDocument = { + _id: { toString(): string } + role?: string + validateSync(): { errors: Record } | undefined +} + +/** + * Data-quality audit (ENG-484 / cutover runbook D1). The pointer-flip step + * saves the whole account document, so mongoose re-validates EVERY field — + * an account with pre-existing invalid data (empty username, null + * statusHistory entry, ...) fails mid-migration even though the cutover never + * touched those fields. Both were hit live in the ENG-461 rehearsal. + * + * Run before `prepare`: every issue reported here is an account that WILL + * fail at pointer flip unless fixed (or locked out of the run). + */ +export const auditCashWalletCutoverAccounts = async ({ + listAccountDocuments = () => + Account.find({}).cursor() as unknown as AsyncIterable, +}: { + listAccountDocuments?: () => AsyncIterable +} = {}): Promise => { + const report: CashWalletCutoverAccountAuditReport = { + scanned: 0, + invalid: 0, + issues: [], + } + + for await (const document of listAccountDocuments()) { + report.scanned += 1 + + const validation = document.validateSync() + if (validation === undefined) continue + + report.invalid += 1 + report.issues.push({ + accountId: document._id.toString(), + role: document.role, + errors: Object.values(validation.errors).map(({ message }) => message), + }) + } + + return report +} diff --git a/src/app/cash-wallet-cutover/handlers.ts b/src/app/cash-wallet-cutover/handlers.ts index 28cdfdc15..f9afef03a 100644 --- a/src/app/cash-wallet-cutover/handlers.ts +++ b/src/app/cash-wallet-cutover/handlers.ts @@ -3,6 +3,7 @@ import { createCashWalletMigrationBalanceMoveInvoice, createCashWalletMigrationFeeReimbursementInvoice, flipCashWalletMigrationDefaultPointer, + isSubMinimumFeeReimbursementAmount, markCashWalletMigrationBalanceMoveSent, markCashWalletMigrationFeeReimbursed, provisionCashWalletMigrationDestination, @@ -131,10 +132,13 @@ export const createCashWalletMigrationStepHandlers = ({ const feeAmountUsdtMicros = await services.feeService.readFeeAmountUsdtMicros(migration) if (feeAmountUsdtMicros instanceof Error) return feeAmountUsdtMicros - if (feeAmountUsdtMicros === "0") { + const subMinimum = isSubMinimumFeeReimbursementAmount(feeAmountUsdtMicros) + if (subMinimum instanceof Error) return subMinimum + if (subMinimum) { return skipCashWalletMigrationFeeReimbursement({ migration, migrationsRepo, + feeAmountUsdtMicros, }) } return createCashWalletMigrationFeeReimbursementInvoice({ diff --git a/src/app/cash-wallet-cutover/index.ts b/src/app/cash-wallet-cutover/index.ts index e7d59d606..02334252c 100644 --- a/src/app/cash-wallet-cutover/index.ts +++ b/src/app/cash-wallet-cutover/index.ts @@ -20,6 +20,8 @@ export * from "./handlers" export * from "./runtime-services" export * from "./rollback-worker" export * from "./rollback" +export * from "./retry-failed" +export * from "./audit-accounts" export * from "./orchestrator" export * from "./lifecycle" export * from "./preview" diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index 982c6ff64..b6832c0e8 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -491,7 +491,7 @@ const parseIntegerAmount = (value?: string): number | undefined => { return Number(value) } -const computeCutoverBalanceAudit = ({ +export const computeCutoverBalanceAudit = ({ migration, usdtWallets, }: { @@ -539,9 +539,18 @@ const computeCutoverBalanceAudit = ({ 0, currentDestinationBalanceUsdtMicros - destinationStartingBalanceUsdtMicros, ) + // ENG-484: an absorbed fee (recorded but never reimbursed — no payment + // transaction id) legitimately leaves the destination short of target by + // exactly that fee. Without this allowance every dust-fee account renders + // as a red "shortfall" row during the post-run reconciliation hold, + // drowning real shortfalls. + const absorbedFeeUsdtMicros = + migration.feeReimbursementPaymentTransactionId === undefined + ? (parseIntegerAmount(migration.feeAmountUsdtMicros) ?? 0) + : 0 const shortfallUsdtMicros = Math.max( 0, - expectedMinimumUsdtMicros - finalDeltaUsdtMicros, + expectedMinimumUsdtMicros - absorbedFeeUsdtMicros - finalDeltaUsdtMicros, ) const roundingSubsidyUsdtMicros = Math.max( 0, diff --git a/src/app/cash-wallet-cutover/retry-failed.ts b/src/app/cash-wallet-cutover/retry-failed.ts new file mode 100644 index 000000000..8b2b5f4e2 --- /dev/null +++ b/src/app/cash-wallet-cutover/retry-failed.ts @@ -0,0 +1,142 @@ +import { CashWalletCutoverRepository } from "@services/mongoose" + +import { CashWalletMigrationFailedError } from "./errors" + +type CashWalletRetryRepository = ReturnType + +export type CashWalletRetryFailedReport = { + dryRun: boolean + eligible: number + retried: number + /** Migrations skipped because money moved — those are operator-review work. */ + skippedMoneyMoved: number + /** Lost races: migrations concurrently moved out of `failed` mid-run. */ + conflicts: number + resumedAt: Partial> + migrationIds: string[] +} + +/** + * Re-drive `failed` migrations (ENG-484). Safe by construction: the runner + * routes failures in money-moving statuses to `requires_operator_review`, so + * anything sitting in `failed` has no ambiguous side effects. Resume point is + * field-driven, mirroring the forward pipeline's guards: + * + * - pointer flipped (`previousDefaultWalletId` set) → resume `pointer_flipped`, + * preserving the recorded pointer so a later rollback stays correct + * - otherwise → reset to `not_started`, clearing stale progress fields so the + * machine re-walks provisioning and balance read from scratch + * + * Migrations with a recorded payment transaction are never retried here — + * defense in depth should one ever land in `failed`. + */ +export const retryFailedCashWalletMigrations = async ({ + cutoverVersion, + runId, + accountId, + dryRun = false, + migrationsRepo = CashWalletCutoverRepository(), +}: { + cutoverVersion: number + runId: string + /** Single-account mode when set; all failed migrations when omitted. */ + accountId?: AccountId + dryRun?: boolean + migrationsRepo?: CashWalletRetryRepository +}): Promise => { + let candidates: CashWalletMigration[] + + if (accountId !== undefined) { + const migration = await migrationsRepo.findMigrationByAccountId({ + accountId, + cutoverVersion, + runId, + }) + if (migration instanceof Error) return migration + if (migration === null) { + return new CashWalletMigrationFailedError( + `No migration found for accountId=${accountId} runId=${runId}`, + ) + } + if (migration.status !== "failed") { + return new CashWalletMigrationFailedError( + `Migration for accountId=${accountId} is ${migration.status}, not failed`, + ) + } + candidates = [migration] + } else { + const migrations = await migrationsRepo.listMigrationsByStatuses({ + cutoverVersion, + runId, + statuses: ["failed"], + }) + if (migrations instanceof Error) return migrations + candidates = migrations + } + + const report: CashWalletRetryFailedReport = { + dryRun, + eligible: 0, + retried: 0, + skippedMoneyMoved: 0, + conflicts: 0, + resumedAt: {}, + migrationIds: [], + } + + for (const migration of candidates) { + if ( + migration.balanceMovePaymentTransactionId !== undefined || + migration.feeReimbursementPaymentTransactionId !== undefined + ) { + report.skippedMoneyMoved += 1 + continue + } + + const resumeStatus: CashWalletMigrationStatus = + migration.previousDefaultWalletId !== undefined ? "pointer_flipped" : "not_started" + + report.eligible += 1 + report.migrationIds.push(migration.id) + report.resumedAt[resumeStatus] = (report.resumedAt[resumeStatus] ?? 0) + 1 + if (dryRun) continue + + const transitioned = await migrationsRepo.transitionMigration({ + id: migration.id, + from: "failed", + to: resumeStatus, + cutoverVersion, + runId, + patch: { attempts: 0 }, + clear: + resumeStatus === "not_started" + ? [ + "lastError", + "sourceBalanceUsdCents", + "destinationAmountUsdtMicros", + "destinationStartingBalanceUsdtMicros", + "feeAmountUsdCents", + "feeAmountUsdtMicros", + "balanceMoveInvoicePaymentRequest", + "balanceMoveInvoicePaymentHash", + "feeReimbursementInvoicePaymentRequest", + "feeReimbursementInvoicePaymentHash", + ] + : ["lastError"], + }) + if (transitioned instanceof Error) { + // Lost a race (e.g. a concurrent rollback-request moved this migration + // out of `failed` — the guarded findOneAndUpdate matched nothing). + // Skip it and keep going: aborting would discard the report of + // migrations already reset by earlier iterations. + report.eligible -= 1 + report.migrationIds.pop() + report.resumedAt[resumeStatus] = (report.resumedAt[resumeStatus] ?? 1) - 1 + report.conflicts += 1 + continue + } + report.retried += 1 + } + + return report +} diff --git a/src/app/cash-wallet-cutover/rollback-worker.ts b/src/app/cash-wallet-cutover/rollback-worker.ts index 03046e721..4e3bade03 100644 --- a/src/app/cash-wallet-cutover/rollback-worker.ts +++ b/src/app/cash-wallet-cutover/rollback-worker.ts @@ -1,6 +1,6 @@ import { assertCanTransition, ROLLBACKABLE_STATUSES } from "./state-machine" import { legacyShortfallUsdtMicros } from "./amount-conversion" -import { isInvoicePaymentRequestStale } from "./worker" +import { CUTOVER_MIN_PAYABLE_USDT_MICROS, isInvoicePaymentRequestStale } from "./worker" import { CashWalletMigrationFailedError, InvalidCashWalletMigrationTransitionError, @@ -40,6 +40,9 @@ type CashWalletRollbackServices = { readDestinationBalanceUsdtMicros( migration: CashWalletMigration, ): Promise + readDestinationSpendableUsdtMicros( + migration: CashWalletMigration, + ): Promise } invoiceService: { createNoAmountInvoice(args: { @@ -194,17 +197,37 @@ export const executeCashWalletMigrationRollbackStep = async ({ } if (current.destinationAmountUsdtMicros !== "0") { - // Fail closed: if the user transacted on the USDT wallet since the - // forward move, reversing the full amount is not possible without - // operator judgment. - const usdtBalance = - await services.balanceReader.readDestinationBalanceUsdtMicros(current) - if (usdtBalance instanceof Error) return usdtBalance - if (BigInt(usdtBalance) < BigInt(current.destinationAmountUsdtMicros)) { + // Reverse the ACTUAL spendable balance (floored), capped at the forward + // amount — never the raw target. The USDT wallet legitimately holds + // slightly less than target: the un-reimbursed forward routing fee plus + // sub-micro rounding. Paying the target verbatim overspends → IBEX 400 + // (ENG-401). Reversing the spendable balance makes the user whole minus + // that sub-cent dust. Fail closed only when the wallet is short by MORE + // than tolerance — the signature of the user having spent USDT + // post-cutover, which needs operator judgment. + const target = BigInt(current.destinationAmountUsdtMicros) + const spendableStr = + await services.balanceReader.readDestinationSpendableUsdtMicros(current) + if (spendableStr instanceof Error) return spendableStr + const spendable = BigInt(spendableStr) + + const reverseAmount = spendable < target ? spendable : target + const shortfall = target - reverseAmount + if (shortfall > ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS) { + return new CashWalletMigrationFailedError( + `USDT balance ${spendableStr} is short of reverse target ${target} by ${shortfall} micros (> tolerance) — likely spent post-cutover`, + ) + } + // IBEX rejects payments below ~1 sat (ENG-484): a drained or 1-cent + // wallet can pass the tolerance gate with a 0- or sub-minimum reverse + // amount — fail closed with a clear message instead of a doomed pay + // attempt surfacing as a misleading IBEX 400. + if (reverseAmount < CUTOVER_MIN_PAYABLE_USDT_MICROS) { return new CashWalletMigrationFailedError( - `USDT balance ${usdtBalance} no longer covers reverse amount ${current.destinationAmountUsdtMicros}`, + `Reverse amount ${reverseAmount} micros is below IBEX's minimum payable — target ${target}, spendable ${spendableStr}; resolve manually`, ) } + const reverseAmountStr = reverseAmount.toString() if ( current.rollbackInvoicePaymentRequest === undefined || @@ -240,7 +263,7 @@ export const executeCashWalletMigrationRollbackStep = async ({ const payment = await services.paymentService.payInvoice({ senderWalletId: current.destinationUsdtWalletId, paymentRequest: current.rollbackInvoicePaymentRequest, - senderAmountUsdtMicros: current.destinationAmountUsdtMicros, + senderAmountUsdtMicros: reverseAmountStr, }) if (payment instanceof Error) return payment @@ -273,6 +296,14 @@ export const executeCashWalletMigrationRollbackStep = async ({ }) if (shortfall instanceof Error) return shortfall + // Accepted boundary tradeoff: step 2 tolerates up to a 1-cent USDT + // shortfall (user spend + un-reimbursed fee), and this top-up covers + // whatever the legacy wallet is short vs the original balance — so up to + // ~1 cent of a user's own post-cutover spending can be reimbursed by the + // treasury (spend ≤ tolerance at step 2, but spend + fee + receive dust + // over it here). Exposure is bounded per account at roughly a cent plus + // fee plus dust; the old strict fail-closed left every such account in + // manual review instead. Documented in ENG-484. if (BigInt(shortfall) > ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS) { if (current.rollbackShortfallPaymentTransactionId !== undefined) { // Treasury already topped up once and the user is still short more diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index abf75e1f0..ce49a7028 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -3,7 +3,7 @@ import { decodeInvoice } from "@domain/bitcoin/lightning" import { InvalidWalletId } from "@domain/errors" import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" import { WalletType } from "@domain/wallets" -import { AccountsRepository } from "@services/mongoose" +import { AccountsRepository, WalletsRepository } from "@services/mongoose" import Ibex from "@services/ibex/client" import { UnexpectedIbexResponse } from "@services/ibex/errors" import { getFunderWalletId } from "@services/ledger/caching" @@ -30,6 +30,8 @@ type RuntimeServiceDependencies = { createNoAmountInvoice?: typeof Ibex.addInvoice payInvoice?: typeof Ibex.payInvoice accountsRepo?: Pick, "findById"> + walletsRepo?: Pick, "findById" | "listByAccountId"> + getFunderWalletId?: typeof getFunderWalletId getTreasuryWalletId?: () => Promise maxRateLimitAttempts?: number rateLimitRetryDelayMs?: number @@ -139,7 +141,18 @@ const withIbexRateLimitRetry = async ({ sleepFn: SleepFn }): Promise => { for (let attempt = 1; ; attempt += 1) { - const result = await operation() + // The IBEX client surfaces rate limits both ways: some call paths return + // an IbexError, others REJECT with a raw FetchError ("Too Many Requests"). + // Retry both — a thrown 429 killed 233 accounts in the ENG-461 rehearsal. + let result: T | Error + try { + result = await operation() + } catch (thrown) { + const error = thrown instanceof Error ? thrown : new Error(String(thrown)) + if (!isIbexRateLimitError(error) || attempt >= maxAttempts) throw thrown + await sleepFn(retryDelayMs) + continue + } if ( !(result instanceof Error) || @@ -163,6 +176,8 @@ export const createCashWalletMigrationRuntimeServices = ( const noAmountInvoiceForRecipient = deps.createNoAmountInvoice ?? Ibex.addInvoice const payInvoice = deps.payInvoice ?? Ibex.payInvoice const accountsRepo = deps.accountsRepo ?? AccountsRepository() + const walletsRepo = deps.walletsRepo ?? WalletsRepository() + const funderWalletId = deps.getFunderWalletId ?? getFunderWalletId const rateLimitRetry = { maxAttempts: Math.max( 1, @@ -171,6 +186,14 @@ export const createCashWalletMigrationRuntimeServices = ( retryDelayMs: deps.rateLimitRetryDelayMs ?? CUTOVER_IBEX_RATE_LIMIT_RETRY_DELAY_MS, sleepFn: deps.sleep ?? sleep, } + // ENG-483: balance reads were the one IBEX call path NOT retried on 429 — + // an unthrottled batch mass-failed 233 accounts at the balance_read step. + // Every account-details read goes through the same retry as invoice/payment. + const readRawAccountDetails = (accountId: IbexAccountId) => + withIbexRateLimitRetry({ + ...rateLimitRetry, + operation: () => rawAccountDetails(accountId), + }) return { now: deps.now ?? (() => new Date()), @@ -207,7 +230,7 @@ export const createCashWalletMigrationRuntimeServices = ( readSourceBalanceUsdCents: async ( migration: CashWalletMigration, ): Promise => { - const account = await rawAccountDetails( + const account = await readRawAccountDetails( migration.legacyUsdWalletId as IbexAccountId, ) if (account instanceof Error) return account @@ -216,12 +239,30 @@ export const createCashWalletMigrationRuntimeServices = ( readDestinationBalanceUsdtMicros: async ( migration: CashWalletMigration, ): Promise => { - const account = await rawAccountDetails( + const account = await readRawAccountDetails( migration.destinationUsdtWalletId as IbexAccountId, ) if (account instanceof Error) return account return ibexMajorUnitsToUsdtMicros(account.balance ?? 0) }, + // Rollback (ENG-401): the FLOORED spendable USDT balance. The rounded + // read above can round UP past what's actually spendable (e.g. balance + // 0.443691959514 → 443692 micros), so paying it back verbatim overspends + // and IBEX 400s. Flooring guarantees the reverse amount never exceeds the + // real balance. + readDestinationSpendableUsdtMicros: async ( + migration: CashWalletMigration, + ): Promise => { + const account = await readRawAccountDetails( + migration.destinationUsdtWalletId as IbexAccountId, + ) + if (account instanceof Error) return account + return decimalToScaledInteger({ + amount: account.balance ?? 0, + scale: 6, + round: false, + }) + }, }, invoiceService: { createInvoice: ({ @@ -315,7 +356,7 @@ export const createCashWalletMigrationRuntimeServices = ( }: { legacyUsdWalletId: WalletId }): Promise => { - const account = await rawAccountDetails(legacyUsdWalletId as IbexAccountId) + const account = await readRawAccountDetails(legacyUsdWalletId as IbexAccountId) if (account instanceof Error) return account if (!hasOnlySubMicroMajorUnitDust(account.balance)) { return new CashWalletMigrationFailedError("Legacy USD wallet is not zero") @@ -345,7 +386,7 @@ export const createCashWalletMigrationRuntimeServices = ( ) } - const currentAccount = await rawAccountDetails( + const currentAccount = await readRawAccountDetails( migration.destinationUsdtWalletId as IbexAccountId, ) if (currentAccount instanceof Error) return currentAccount @@ -361,7 +402,53 @@ export const createCashWalletMigrationRuntimeServices = ( }, }, treasuryService: { - getTreasuryWalletId: deps.getTreasuryWalletId ?? getFunderWalletId, + // ENG-482: the treasury must be a USDT wallet — fee reimbursements and + // rollback top-ups pay USDT invoices, and paying them from the funder's + // BTC default 404s at IBEX (stranded every funded account in the ENG-461 + // rehearsal). Resolve the funder's USDT wallet regardless of which + // wallet is its default, so prod doesn't depend on flipping the funder + // default as an operational step. + // Memoized: the id is invariant for the life of the run and this is + // called per migration at the fee step — without the cache a transient + // mongo blip mid-run fails a step the old memoized getFunderWalletId + // never would have. + getTreasuryWalletId: + deps.getTreasuryWalletId ?? + (() => { + let cached: WalletId | undefined + return async (): Promise => { + if (cached !== undefined) return cached + const funderDefaultWalletId = await funderWalletId() + const funderWallet = await walletsRepo.findById(funderDefaultWalletId) + if (funderWallet instanceof Error) return funderWallet + if (funderWallet.currency === WalletCurrency.Usdt) { + cached = funderWallet.id + return cached + } + + const funderWallets = await walletsRepo.listByAccountId( + funderWallet.accountId, + ) + if (funderWallets instanceof Error) return funderWallets + const usdtWallets = funderWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usdt, + ) + if (usdtWallets.length === 0) { + return new CashWalletMigrationFailedError( + "Funder account has no USDT wallet — create and fund the cutover treasury before running the cutover (ENG-482)", + ) + } + if (usdtWallets.length > 1) { + // No unique index enforces one-USDT-wallet-per-account; picking + // one blind risks paying from an unfunded duplicate. + return new CashWalletMigrationFailedError( + `Funder account has ${usdtWallets.length} USDT wallets — resolve the duplicate before running the cutover (ENG-482)`, + ) + } + cached = usdtWallets[0].id + return cached + } + })(), }, pointerService: { flipDefaultWallet: async ({ @@ -390,7 +477,7 @@ export const createCashWalletMigrationRuntimeServices = ( }: { legacyUsdWalletId: WalletId }): Promise => { - const account = await rawAccountDetails(legacyUsdWalletId as IbexAccountId) + const account = await readRawAccountDetails(legacyUsdWalletId as IbexAccountId) if (account instanceof Error) return account if (!hasOnlySubMicroMajorUnitDust(account.balance)) { return new CashWalletMigrationFailedError("Legacy USD wallet is not zero") diff --git a/src/app/cash-wallet-cutover/state-machine.ts b/src/app/cash-wallet-cutover/state-machine.ts index b6ff9158e..02dc6714f 100644 --- a/src/app/cash-wallet-cutover/state-machine.ts +++ b/src/app/cash-wallet-cutover/state-machine.ts @@ -75,7 +75,16 @@ const transitions: Partial< pointer_flipped: ["legacy_zero_verified", "failed", "rollback_started"], legacy_zero_verified: ["complete", "failed", "rollback_started"], complete: ["rollback_started"], - failed: ["rollback_started"], + // Operator retry (ENG-484). NOTE: `failed` CAN hold money-moved migrations — + // a failure during legacy_zero_verified (balance moved AND pointer flipped) + // routes to `failed` because that status is not in the runner's + // AMBIGUOUS_SIDE_EFFECT_STATUSES (kept that way deliberately: those + // failures are usually transient verify reads and stay auto-retryable). + // These edges are only safe behind retry-failed's guards: skip any + // migration with a recorded payment transaction id, and resume at + // `pointer_flipped` (never `not_started`) when previousDefaultWalletId + // is set. + failed: ["rollback_started", "not_started", "pointer_flipped"], requires_operator_review: ["rollback_started"], // Self-loop persists sub-step artifacts (invoice/payment ids) mid-rollback, // mirroring invoice_created's refresh self-loop. Rollback failures always diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index 2e48e0765..3cfd4bcc6 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -91,6 +91,28 @@ const usesNoAmountFeeReimbursementInvoice = ( return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS } +// ENG-484: IBEX rejects Lightning payments below ~1 satoshi. Measured on TEST +// (2026-07-05, USDT→USDT no-amount invoices): 600 micros → 400 Bad Request, +// 650+ → paid. A 1-sat floor FLOATS with BTC price, so this margin covers it +// up to ~$250k/BTC. Fees below this are absorbed instead of reimbursed +// (≤ ¼ cent — invisible to users); fees above it still pay via the no-amount +// invoice path (which handles 1–9999 micros, e.g. the 4735-micro case below). +// Deliberately NOT tied to CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS: that is an +// invoice-format constant, this is a payability floor + customer-funds policy. +export const CUTOVER_MIN_PAYABLE_USDT_MICROS = 2_500n + +export const isSubMinimumFeeReimbursementAmount = ( + feeAmountUsdtMicros: string, +): boolean | InvalidCashWalletCutoverAmountError => { + if (!/^\d+$/.test(feeAmountUsdtMicros)) { + return new InvalidCashWalletCutoverAmountError( + `Invalid non-negative integer amount: ${feeAmountUsdtMicros}`, + ) + } + + return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_PAYABLE_USDT_MICROS +} + export const isInvoicePaymentRequestStale = ({ paymentRequest, now, @@ -417,9 +439,11 @@ export const createCashWalletMigrationFeeReimbursementInvoice = async ({ export const skipCashWalletMigrationFeeReimbursement = async ({ migration, migrationsRepo, + feeAmountUsdtMicros = "0", }: { migration: CashWalletMigration migrationsRepo: CashWalletMigrationTransitionRepository + feeAmountUsdtMicros?: string }): Promise => { const transition = assertCanTransition(migration.status, "fee_reimbursed") if (transition instanceof Error) return transition @@ -431,8 +455,10 @@ export const skipCashWalletMigrationFeeReimbursement = async ({ cutoverVersion: migration.cutoverVersion, runId: migration.runId, patch: { + // A skipped fee is always sub-cent, so cents are 0; the micros record + // what was absorbed (no feeReimbursementPaymentTransactionId = unpaid). feeAmountUsdCents: "0", - feeAmountUsdtMicros: "0", + feeAmountUsdtMicros, }, }) } diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 1d5762f52..0a45500f7 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -15,6 +15,10 @@ import { baseLogger } from "@services/logger" const args = yargs(hideBin(process.argv)) .command("preview", "discover accounts and print the migration plan without writes") + .command( + "audit-accounts", + "validate every account document; reports accounts that would fail the pointer-flip save (run before prepare)", + ) .command( "provision-usdt-wallets", "create missing destination USDT wallets before preparing migrations", @@ -22,6 +26,10 @@ const args = yargs(hideBin(process.argv)) .command("prepare", "discover accounts and upsert migration records") .command("start", "mark a prepared cutover run in progress") .command("run-batch", "run one locked migration worker batch") + .command( + "retry-failed", + "reset failed migrations to their last safe status for re-running (single account with --account-id, else all failed)", + ) .command("status", "print cutover config and migration counts") .command("complete", "mark cutover complete after all migrations finish") .command( @@ -39,7 +47,10 @@ const args = yargs(hideBin(process.argv)) .option("operator", { type: "string", default: "unknown" }) .option("worker-id", { type: "string", default: `worker-${process.pid}` }) .option("limit", { type: "number", default: 25 }) - .option("step-delay-ms", { type: "number", default: 0 }) + // ENG-483: default throttle ≈30 accounts/min — the empirically safe IBEX + // rate from the ENG-461 rehearsal (0 = unthrottled mass-failed 233 accounts + // on 429s). Pass --step-delay-ms 0 explicitly to disable. + .option("step-delay-ms", { type: "number", default: 2_000 }) .option("provision-limit", { type: "number" }) .option("provision-delay-ms", { type: "number", default: 12_500 }) .option("provision-retry-delay-ms", { type: "number", default: 60_000 }) @@ -73,6 +84,12 @@ const run = async () => { return } + case "audit-accounts": { + const result = await CashWalletCutover.auditCashWalletCutoverAccounts() + toJson(result) + return + } + case "provision-usdt-wallets": { const result = await CashWalletCutover.provisionPrimaryCashWalletUsdtWallets({ cutoverVersion, @@ -160,6 +177,19 @@ const run = async () => { return } + case "retry-failed": { + const result = await CashWalletCutover.retryFailedCashWalletMigrations({ + cutoverVersion, + runId, + accountId: args["account-id"] as AccountId | undefined, + dryRun: args["dry-run"], + migrationsRepo: repository, + }) + if (result instanceof Error) throw result + toJson(result) + return + } + case "rollback-request": { if (!args.reason) { throw new Error("--reason is required for rollback-request") diff --git a/src/services/mongoose/cash-wallet-cutover.ts b/src/services/mongoose/cash-wallet-cutover.ts index d0ce9f4c7..92297524a 100644 --- a/src/services/mongoose/cash-wallet-cutover.ts +++ b/src/services/mongoose/cash-wallet-cutover.ts @@ -34,6 +34,8 @@ type TransitionMigrationArgs = { cutoverVersion: number runId: string patch?: Partial> + /** Fields to $unset — e.g. clearing stale progress on a retry reset (ENG-484). */ + clear?: (keyof Omit)[] } type LockMigrationArgs = { @@ -208,11 +210,17 @@ export const CashWalletCutoverRepository = () => { cutoverVersion, runId, patch = {}, + clear = [], }: TransitionMigrationArgs): Promise => { try { const result = await CashWalletMigration.findOneAndUpdate( { _id: id, status: from, cutoverVersion, runId }, - { $set: { ...patch, status: to, updatedAt: new Date() } }, + { + $set: { ...patch, status: to, updatedAt: new Date() }, + ...(clear.length > 0 + ? { $unset: Object.fromEntries(clear.map((field) => [field, ""])) } + : {}), + }, { new: true }, ) if (!result) diff --git a/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts b/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts new file mode 100644 index 000000000..0c81d877d --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/audit-accounts.spec.ts @@ -0,0 +1,69 @@ +import { auditCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/audit-accounts" + +const doc = ({ + id, + role, + errors, +}: { + id: string + role?: string + errors?: Record +}) => ({ + _id: { toString: () => id }, + role, + validateSync: () => (errors ? { errors } : undefined), +}) + +async function* documents(...docs: ReturnType[]) { + yield* docs +} + +describe("auditCashWalletCutoverAccounts", () => { + it("reports accounts whose document fails mongoose validation", async () => { + const report = await auditCashWalletCutoverAccounts({ + listAccountDocuments: () => + documents( + doc({ id: "good-account" }), + doc({ + id: "empty-username", + errors: { + username: { + message: + "Path `username` (``) is shorter than the minimum allowed length (3).", + }, + }, + }), + doc({ + id: "null-status", + role: "funder", + errors: { + "statusHistory.0.status": { message: "Path `status` is required." }, + }, + }), + ), + }) + + expect(report.scanned).toBe(3) + expect(report.invalid).toBe(2) + expect(report.issues).toEqual([ + { + accountId: "empty-username", + role: undefined, + errors: [expect.stringMatching(/username/)], + }, + { + accountId: "null-status", + role: "funder", + errors: ["Path `status` is required."], + }, + ]) + }) + + it("reports a clean sweep when every account validates", async () => { + const report = await auditCashWalletCutoverAccounts({ + listAccountDocuments: () => documents(doc({ id: "a" }), doc({ id: "b" })), + }) + + expect(report).toEqual({ scanned: 2, invalid: 0, issues: [] }) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts b/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts new file mode 100644 index 000000000..bdbb08fc7 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/balance-audit.spec.ts @@ -0,0 +1,59 @@ +import { computeCutoverBalanceAudit } from "@app/cash-wallet-cutover/operator-dashboard" + +const migration = ({ + feePaid, + feeAmountUsdtMicros = "515", +}: { + feePaid: boolean + feeAmountUsdtMicros?: string +}) => + ({ + id: "migration-id", + status: "complete", + sourceBalanceUsdCents: "100", + destinationAmountUsdtMicros: "10000", + destinationStartingBalanceUsdtMicros: "0", + destinationUsdtWalletId: "dest-wallet" as WalletId, + feeAmountUsdtMicros, + ...(feePaid ? { feeReimbursementPaymentTransactionId: "fee-txn" } : {}), + }) as CashWalletMigration + +const wallet = (minorUnitsNumber: number) => + ({ + id: "dest-wallet", + balance: { status: "fresh", minorUnitsNumber }, + }) as never + +describe("computeCutoverBalanceAudit — absorbed-fee allowance (ENG-484)", () => { + it("does not flag an absorbed fee as a shortfall", () => { + // Fee recorded but never reimbursed: destination legitimately holds + // target − fee. Must render verified, not a red shortfall row. + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: false }), + usdtWallets: [wallet(9485)], // 10000 − 515 absorbed + }) + + expect(audit?.status).toBe("verified") + expect(audit?.shortfallUsdtMicros).toBe(0) + }) + + it("still flags a real shortfall beyond the absorbed fee", () => { + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: false }), + usdtWallets: [wallet(8000)], // short 1485 beyond the absorbed 515 + }) + + expect(audit?.status).toBe("shortfall") + expect(audit?.shortfallUsdtMicros).toBe(1485) + }) + + it("grants no allowance when the fee was actually reimbursed", () => { + const audit = computeCutoverBalanceAudit({ + migration: migration({ feePaid: true, feeAmountUsdtMicros: "5000" }), + usdtWallets: [wallet(9000)], + }) + + expect(audit?.status).toBe("shortfall") + expect(audit?.shortfallUsdtMicros).toBe(1000) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts new file mode 100644 index 000000000..ee92628f3 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/retry-failed.spec.ts @@ -0,0 +1,150 @@ +import { retryFailedCashWalletMigrations } from "@app/cash-wallet-cutover/retry-failed" + +const baseMigration = { + id: "migration-id", + accountId: "account-id" as AccountId, + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, + cutoverVersion: 1, + runId: "run-id", + status: "failed", + idempotencyKey: "run-id:account-id", + attempts: 3, + lastError: "FetchError: Too Many Requests", + updatedAt: new Date("2026-07-05T00:00:00.000Z"), +} as CashWalletMigration + +const repo = (migrations: CashWalletMigration[]) => ({ + findMigrationByAccountId: jest.fn(async ({ accountId }) => { + return migrations.find((m) => m.accountId === accountId) ?? null + }), + listMigrationsByStatuses: jest.fn(async () => migrations), + transitionMigration: jest.fn(async ({ to, patch }) => ({ + ...migrations[0], + ...patch, + status: to, + })), +}) + +describe("retryFailedCashWalletMigrations", () => { + it("resets a pre-money failure to not_started, clearing stale progress", async () => { + const migrationsRepo = repo([ + { ...baseMigration, sourceBalanceUsdCents: "100" } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ + eligible: 1, + retried: 1, + skippedMoneyMoved: 0, + resumedAt: { not_started: 1 }, + }) + expect(migrationsRepo.transitionMigration).toHaveBeenCalledWith( + expect.objectContaining({ + from: "failed", + to: "not_started", + patch: { attempts: 0 }, + clear: expect.arrayContaining(["lastError", "sourceBalanceUsdCents"]), + }), + ) + }) + + it("resumes a post-pointer failure at pointer_flipped, preserving the pointer", async () => { + const migrationsRepo = repo([ + { + ...baseMigration, + previousDefaultWalletId: "legacy-wallet-id" as WalletId, + } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ retried: 1, resumedAt: { pointer_flipped: 1 } }) + const call = migrationsRepo.transitionMigration.mock.calls[0][0] + expect(call.to).toBe("pointer_flipped") + expect(call.clear).toEqual(["lastError"]) + expect(call.clear).not.toContain("previousDefaultWalletId") + }) + + it("never retries a failure with a recorded payment transaction", async () => { + const migrationsRepo = repo([ + { + ...baseMigration, + balanceMovePaymentTransactionId: "txn-id", + } as CashWalletMigration, + ]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ eligible: 0, retried: 0, skippedMoneyMoved: 1 }) + expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() + }) + + it("dry-run reports without writing", async () => { + const migrationsRepo = repo([baseMigration]) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + dryRun: true, + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ dryRun: true, eligible: 1, retried: 0 }) + expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() + }) + + it("skips a lost race and keeps the report of migrations already reset", async () => { + const second = { + ...baseMigration, + id: "migration-2", + accountId: "account-2" as AccountId, + } as CashWalletMigration + const migrationsRepo = repo([baseMigration, second]) + migrationsRepo.transitionMigration + .mockResolvedValueOnce({ ...baseMigration, status: "not_started" }) + .mockResolvedValueOnce(new Error("Could not transition cash wallet migration")) + + const report = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + migrationsRepo: migrationsRepo as never, + }) + + expect(report).toMatchObject({ + retried: 1, + conflicts: 1, + eligible: 1, + migrationIds: ["migration-id"], + resumedAt: { not_started: 1 }, + }) + }) + + it("rejects single-account retry when the migration is not failed", async () => { + const migrationsRepo = repo([ + { ...baseMigration, status: "complete" } as CashWalletMigration, + ]) + + const result = await retryFailedCashWalletMigrations({ + cutoverVersion: 1, + runId: "run-id", + accountId: "account-id" as AccountId, + migrationsRepo: migrationsRepo as never, + }) + + expect(result).toBeInstanceOf(Error) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts index 219cfea93..8e72584a3 100644 --- a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts @@ -45,10 +45,12 @@ const statefulRepo = (initial: CashWalletMigration) => { const services = ({ defaultWalletId = "legacy-wallet-id" as WalletId, usdtBalanceMicros = "0", + spendableUsdtMicros, legacyBalanceUsdCents = "0", }: { defaultWalletId?: WalletId usdtBalanceMicros?: string + spendableUsdtMicros?: string legacyBalanceUsdCents?: string } = {}) => ({ now: () => NOW, @@ -63,6 +65,9 @@ const services = ({ balanceReader: { readSourceBalanceUsdCents: jest.fn().mockResolvedValue(legacyBalanceUsdCents), readDestinationBalanceUsdtMicros: jest.fn().mockResolvedValue(usdtBalanceMicros), + readDestinationSpendableUsdtMicros: jest + .fn() + .mockResolvedValue(spendableUsdtMicros ?? usdtBalanceMicros), }, invoiceService: { createNoAmountInvoice: jest.fn().mockResolvedValue(invoice), @@ -224,6 +229,58 @@ describe("rollback executor", () => { }) }) + it("reverses the SPENDABLE balance (not the target) when short by only dust (ENG-401)", async () => { + // The wallet holds target minus the un-reimbursed forward fee + rounding + // (well within the 1-cent tolerance). Reverse the actual spendable amount, + // never the target — paying the target would overspend and IBEX 400s. + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + } + const svc = services({ + spendableUsdtMicros: "1499996473", // 3527 micros short = forward fee + legacyBalanceUsdCents: "150000", // whole again after the reverse pay + }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(svc.paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: migration.destinationUsdtWalletId, + paymentRequest: invoice.paymentRequest, + senderAmountUsdtMicros: "1499996473", // the spendable balance, not 1500000000 + }) + expect(result).toMatchObject({ status: "rolled_back" }) + }) + + it("fails closed when the reverse amount is below IBEX's minimum payable (ENG-484)", async () => { + // A 1-cent account with a nearly-drained wallet passes the tolerance gate + // (shortfall ≤ 1 cent) but the reverse amount is unpayable — fail closed + // with a clear message instead of a doomed pay and a misleading IBEX 400. + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "1", + destinationAmountUsdtMicros: "10000", + } + const svc = services({ spendableUsdtMicros: "2000" }) // below 2500 min payable + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/below IBEX's minimum payable/) + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + }) + it("fails closed when the USDT balance no longer covers the reverse amount", async () => { const migration = { ...baseMigration, @@ -231,7 +288,7 @@ describe("rollback executor", () => { sourceBalanceUsdCents: "150000", destinationAmountUsdtMicros: "1500000000", } - const svc = services({ usdtBalanceMicros: "900000000" }) // user spent funds + const svc = services({ spendableUsdtMicros: "900000000" }) // user spent funds const result = await executeCashWalletMigrationRollbackStep({ migration, diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts new file mode 100644 index 000000000..3dc469e49 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -0,0 +1,198 @@ +import { createCashWalletMigrationRuntimeServices } from "@app/cash-wallet-cutover/runtime-services" + +const migration = { + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, +} as CashWalletMigration + +const rateLimitFetchError = () => new Error("FetchError: Too Many Requests") + +describe("cutover runtime services — IBEX rate-limit retry (ENG-483)", () => { + it("retries balance reads when the read REJECTS with a 429 FetchError", async () => { + const sleeps: number[] = [] + const getRawAccountDetails = jest + .fn() + .mockRejectedValueOnce(rateLimitFetchError()) + .mockRejectedValueOnce(rateLimitFetchError()) + .mockResolvedValue({ balance: 12.5 }) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 7, + sleep: async (ms) => { + sleeps.push(ms) + }, + }) + + const result = await services.balanceReader.readSourceBalanceUsdCents(migration) + + expect(result).toBe("1250") + expect(getRawAccountDetails).toHaveBeenCalledTimes(3) + expect(sleeps).toEqual([7, 7]) + }) + + it("retries balance reads when the read RETURNS a rate-limit error", async () => { + const getRawAccountDetails = jest + .fn() + .mockResolvedValueOnce(new Error("Too Many Requests")) + .mockResolvedValue({ balance: 3 }) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + const result = + await services.balanceReader.readDestinationBalanceUsdtMicros(migration) + + expect(result).toBe("3000000") + expect(getRawAccountDetails).toHaveBeenCalledTimes(2) + }) + + it("gives up after maxAttempts and rethrows the final 429", async () => { + const getRawAccountDetails = jest.fn().mockRejectedValue(rateLimitFetchError()) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + maxRateLimitAttempts: 3, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + await expect( + services.balanceReader.readSourceBalanceUsdCents(migration), + ).rejects.toThrow("Too Many Requests") + expect(getRawAccountDetails).toHaveBeenCalledTimes(3) + }) + + it("resolves the treasury to the funder's USDT wallet regardless of its default (ENG-482)", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest.fn().mockResolvedValue([ + { id: "funder-btc-wallet", currency: "BTC" }, + { id: "funder-usdt-wallet", currency: "USDT" }, + ]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + await expect(services.treasuryService.getTreasuryWalletId()).resolves.toBe( + "funder-usdt-wallet", + ) + }) + + it("short-circuits when the funder default is already the USDT wallet", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-usdt-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "USDT", + }), + listByAccountId: jest.fn(), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-usdt-wallet" as WalletId, + }) + + await expect(services.treasuryService.getTreasuryWalletId()).resolves.toBe( + "funder-usdt-wallet", + ) + expect(walletsRepo.listByAccountId).not.toHaveBeenCalled() + }) + + it("memoizes treasury resolution — one lookup for the whole run", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-usdt-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "USDT", + }), + listByAccountId: jest.fn(), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-usdt-wallet" as WalletId, + }) + + await services.treasuryService.getTreasuryWalletId() + await services.treasuryService.getTreasuryWalletId() + await services.treasuryService.getTreasuryWalletId() + + expect(walletsRepo.findById).toHaveBeenCalledTimes(1) + }) + + it("fails closed when the funder has duplicate USDT wallets", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest.fn().mockResolvedValue([ + { id: "usdt-1", currency: "USDT" }, + { id: "usdt-2", currency: "USDT" }, + ]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + const result = await services.treasuryService.getTreasuryWalletId() + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/2 USDT wallets/) + }) + + it("returns a descriptive error when the funder has no USDT wallet", async () => { + const walletsRepo = { + findById: jest.fn().mockResolvedValue({ + id: "funder-btc-wallet" as WalletId, + accountId: "funder-account" as AccountId, + currency: "BTC", + }), + listByAccountId: jest + .fn() + .mockResolvedValue([{ id: "funder-btc-wallet", currency: "BTC" }]), + } + + const services = createCashWalletMigrationRuntimeServices({ + walletsRepo: walletsRepo as never, + getFunderWalletId: async () => "funder-btc-wallet" as WalletId, + }) + + const result = await services.treasuryService.getTreasuryWalletId() + expect(result).toBeInstanceOf(Error) + expect((result as Error).message).toMatch(/no USDT wallet/) + }) + + it("does not retry non-rate-limit rejections", async () => { + const getRawAccountDetails = jest + .fn() + .mockRejectedValue(new Error("FetchError: Not Found")) + + const services = createCashWalletMigrationRuntimeServices({ + getRawAccountDetails, + rateLimitRetryDelayMs: 1, + sleep: async () => undefined, + }) + + await expect( + services.balanceVerifier.verifyBalanceMove({ + legacyUsdWalletId: migration.legacyUsdWalletId, + }), + ).rejects.toThrow("Not Found") + expect(getRawAccountDetails).toHaveBeenCalledTimes(1) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts index 32ffd382e..1396268d8 100644 --- a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts @@ -1,6 +1,8 @@ import { createCashWalletMigrationFeeReimbursementInvoice, + isSubMinimumFeeReimbursementAmount, sendCashWalletMigrationFeeReimbursementPayment, + skipCashWalletMigrationFeeReimbursement, } from "@app/cash-wallet-cutover/worker" const migration = { @@ -90,4 +92,39 @@ describe("cash wallet migration worker fee reimbursement", () => { senderAmountUsdtMicros: "4735", }) }) + + it("classifies only sub-minimum-payable fees as absorbed", () => { + expect(isSubMinimumFeeReimbursementAmount("0")).toBe(true) + expect(isSubMinimumFeeReimbursementAmount("515")).toBe(true) // rehearsal dust (observed 400) + expect(isSubMinimumFeeReimbursementAmount("2499")).toBe(true) + expect(isSubMinimumFeeReimbursementAmount("2500")).toBe(false) + // payable via the no-amount invoice path — NOT absorbed (reviewer: the + // 2500–9999 band reimbursed successfully before ENG-484) + expect(isSubMinimumFeeReimbursementAmount("4735")).toBe(false) + expect(isSubMinimumFeeReimbursementAmount("9999")).toBe(false) + expect(isSubMinimumFeeReimbursementAmount("-1")).toBeInstanceOf(Error) + }) + + it("records the absorbed micros when skipping a dust fee (ENG-484)", async () => { + const migrationsRepo = transitionRepo() + + const result = await skipCashWalletMigrationFeeReimbursement({ + migration, + migrationsRepo, + feeAmountUsdtMicros: "515", + }) + + expect(result).toEqual({ + ...migration, + status: "fee_reimbursed", + feeAmountUsdCents: "0", + feeAmountUsdtMicros: "515", + }) + expect(migrationsRepo.transitionMigration).toHaveBeenCalledWith( + expect.objectContaining({ + to: "fee_reimbursed", + patch: { feeAmountUsdCents: "0", feeAmountUsdtMicros: "515" }, + }), + ) + }) })