From 9d2abb75b36efa30924b70e69b0456cbe86d26ab Mon Sep 17 00:00:00 2001 From: Dread Date: Sat, 4 Jul 2026 11:43:38 -0400 Subject: [PATCH 1/2] feat(cutover): implement rollback handlers, reverse payment, and admin surface (ENG-401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the ENG-364 rollback contingency plan executable. The state machine had rollback_started/rolled_back statuses but nothing could enter or process them — the runbook's "roll back from snapshot on 1-cent drift" promise was unexecutable. State machine: every forward/failed/review/complete status may now enter rollback_started (skipped_already_migrated and rolled_back excluded by design); rollback_started self-loops to persist sub-step artifacts and fails closed into requires_operator_review. Rollback worker (rollback-worker.ts): field-driven resumable executor mirroring the forward pipeline's field-presence guards — 1. restore default-wallet pointer (previousDefaultWalletId, captured at forward flip; legacy USD wallet as constructive fallback) 2. reverse the balance move: no-amount invoice on the legacy USD wallet paid from the USDT wallet with the exact forward amount (destinationAmountUsdtMicros); fails closed if the USDT balance no longer covers it (user transacted post-cutover) 3. treasury tops up any legacy USD shortfall vs sourceBalanceUsdCents (1-cent dust tolerance; hard no-double-pay guard) 4. finalize to rolled_back with rolledBackAt Pre-money migrations short-circuit straight to rolled_back. Reverse payments traced by account, migration id, run id (cwco-rb memos), and operator (rollbackRequestedBy/reason audit fields). Orchestration (rollback.ts): requestPrimaryCashWalletRollback (single account or whole run, dry-run, idempotent — already-rolled-back are skips), runPrimaryCashWalletRollbackBatch (reuses forward batch locking/stale-lock recovery; all failures marked requires_operator_review), completePrimaryCashWalletRollback (flips the cutover config to the new rolled_back state only when nothing is mid-flight or in review). Admin GraphQL: cashWalletCutoverRollback mutation (admin-only), cashWalletMigrations query exposing full migration records incl. previousDefaultWalletId and the rollback audit trail, ROLLED_BACK added to CashWalletCutoverState (additive; public SDL change is enum-value only). CLI: rollback-request / rollback-batch / rollback-complete. Tests: 11 new unit tests (rollback-worker.spec.ts) covering request semantics, every executor path (short-circuit, pointer restore, reverse pay, fail-closed insufficiency, resume, shortfall top-up, dust tolerance, no-double-pay), and decimal-cent shortfall math. tsc + eslint clean; all 7 cutover suites pass (34 tests). Linear: ENG-401 (implements ENG-364 modes 3A/3C/3D) Co-Authored-By: Claude Opus 4.8 (1M context) --- dev/apollo-federation/supergraph.graphql | 1 + .../cash-wallet-cutover/amount-conversion.ts | 23 ++ src/app/cash-wallet-cutover/index.ts | 2 + src/app/cash-wallet-cutover/index.types.d.ts | 18 +- .../cash-wallet-cutover/rollback-worker.ts | 343 +++++++++++++++++ src/app/cash-wallet-cutover/rollback.ts | 223 +++++++++++ .../cash-wallet-cutover/runtime-services.ts | 11 + src/app/cash-wallet-cutover/state-machine.ts | 77 +++- src/app/cash-wallet-cutover/worker.ts | 2 +- src/graphql/admin/mutations.ts | 2 + src/graphql/admin/queries.ts | 2 + .../mutation/cash-wallet-cutover-rollback.ts | 79 ++++ .../root/query/cash-wallet-migrations.ts | 59 +++ src/graphql/admin/schema.graphql | 52 +++ .../types/object/cash-wallet-migration.ts | 44 +++ src/graphql/public/schema.graphql | 1 + .../types/scalar/cash-wallet-cutover-state.ts | 3 + src/scripts/cash-wallet-cutover.ts | 56 +++ src/services/mongoose/cash-wallet-cutover.ts | 42 +++ src/services/mongoose/schema.ts | 13 + src/services/mongoose/schema.types.d.ts | 13 + .../rollback-worker.spec.ts | 351 ++++++++++++++++++ 22 files changed, 1404 insertions(+), 13 deletions(-) create mode 100644 src/app/cash-wallet-cutover/rollback-worker.ts create mode 100644 src/app/cash-wallet-cutover/rollback.ts create mode 100644 src/graphql/admin/root/mutation/cash-wallet-cutover-rollback.ts create mode 100644 src/graphql/admin/root/query/cash-wallet-migrations.ts create mode 100644 src/graphql/admin/types/object/cash-wallet-migration.ts create mode 100644 test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts diff --git a/dev/apollo-federation/supergraph.graphql b/dev/apollo-federation/supergraph.graphql index 26fb4de6f..3f788728c 100644 --- a/dev/apollo-federation/supergraph.graphql +++ b/dev/apollo-federation/supergraph.graphql @@ -619,6 +619,7 @@ enum CashWalletCutoverState COMPLETE @join__enumValue(graph: PUBLIC) IN_PROGRESS @join__enumValue(graph: PUBLIC) PRE @join__enumValue(graph: PUBLIC) + ROLLED_BACK @join__enumValue(graph: PUBLIC) } """(Positive) Cent amount (1/100 of a dollar)""" diff --git a/src/app/cash-wallet-cutover/amount-conversion.ts b/src/app/cash-wallet-cutover/amount-conversion.ts index 9d503f70b..f8c7996ca 100644 --- a/src/app/cash-wallet-cutover/amount-conversion.ts +++ b/src/app/cash-wallet-cutover/amount-conversion.ts @@ -53,6 +53,29 @@ export const usdtMicrosToUsdCentsCeil = ( return ((parsed + USDT_MICROS_PER_USD_CENT - 1n) / USDT_MICROS_PER_USD_CENT).toString() } +// Rollback (ENG-401): how much USD (expressed in USDT micros so it can be +// paid with a sender-side USDT amount) the legacy wallet is still missing +// versus the balance the account originally migrated with. Both inputs are +// precise-cent decimal strings (up to 6 decimal places, as produced by +// ibexUsdDollarsToPreciseCents). +export const legacyShortfallUsdtMicros = ({ + sourceUsdCents, + currentUsdCents, +}: { + sourceUsdCents: string + currentUsdCents: string +}): string | InvalidCashWalletCutoverAmountError => { + const source = usdCentsToUsdtMicros(sourceUsdCents) + if (source instanceof Error) return source + const current = usdCentsToUsdtMicros(currentUsdCents) + if (current instanceof Error) return current + + const sourceMicros = BigInt(source) + const currentMicros = BigInt(current) + if (currentMicros >= sourceMicros) return "0" + return (sourceMicros - currentMicros).toString() +} + export const destinationShortfallUsdtMicros = ({ targetUsdtMicros, startingUsdtMicros, diff --git a/src/app/cash-wallet-cutover/index.ts b/src/app/cash-wallet-cutover/index.ts index 89ad096c5..e7d59d606 100644 --- a/src/app/cash-wallet-cutover/index.ts +++ b/src/app/cash-wallet-cutover/index.ts @@ -18,6 +18,8 @@ export * from "./executor" export * from "./runner" export * from "./handlers" export * from "./runtime-services" +export * from "./rollback-worker" +export * from "./rollback" export * from "./orchestrator" export * from "./lifecycle" export * from "./preview" diff --git a/src/app/cash-wallet-cutover/index.types.d.ts b/src/app/cash-wallet-cutover/index.types.d.ts index 3f16650b7..150aeeb61 100644 --- a/src/app/cash-wallet-cutover/index.types.d.ts +++ b/src/app/cash-wallet-cutover/index.types.d.ts @@ -1,4 +1,4 @@ -type CashWalletCutoverState = "pre" | "in_progress" | "complete" +type CashWalletCutoverState = "pre" | "in_progress" | "complete" | "rolled_back" type CashWalletMigrationStatus = | "not_started" @@ -64,4 +64,20 @@ type CashWalletMigration = { startedAt?: Date completedAt?: Date updatedAt: Date + // Rollback (ENG-401). Progress inside `rollback_started` is field-driven — + // each sub-step records its artifact and is skipped on resume, mirroring + // how the forward pipeline guards on field presence. + rollbackRequestedAt?: Date + rollbackRequestedBy?: string + rollbackReason?: string + rollbackFromStatus?: CashWalletMigrationStatus + rollbackPointerRestoredAt?: Date + rollbackInvoicePaymentRequest?: string + rollbackInvoicePaymentHash?: string + rollbackPaymentTransactionId?: string + rollbackShortfallUsdtMicros?: string + rollbackShortfallInvoicePaymentRequest?: string + rollbackShortfallInvoicePaymentHash?: string + rollbackShortfallPaymentTransactionId?: string + rolledBackAt?: Date } diff --git a/src/app/cash-wallet-cutover/rollback-worker.ts b/src/app/cash-wallet-cutover/rollback-worker.ts new file mode 100644 index 000000000..9587ab814 --- /dev/null +++ b/src/app/cash-wallet-cutover/rollback-worker.ts @@ -0,0 +1,343 @@ +import { assertCanTransition, ROLLBACKABLE_STATUSES } from "./state-machine" +import { legacyShortfallUsdtMicros } from "./amount-conversion" +import { isInvoicePaymentRequestStale } from "./worker" +import { + CashWalletMigrationFailedError, + InvalidCashWalletMigrationTransitionError, +} from "./errors" + +// One US cent, in USDT micros. Reverse moves and treasury top-ups are LN +// payments whose received amount can differ from the sent amount by routing +// dust; anything at or under a cent is treated as whole. +const ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS = 10_000n + +type CashWalletMigrationTransitionRepository = { + transitionMigration(args: { + id: string + from: CashWalletMigrationStatus + to: CashWalletMigrationStatus + cutoverVersion: number + runId: string + patch?: Partial + }): Promise +} + +type CashWalletRollbackServices = { + now(): Date + accountReader: { + getDefaultWalletId(accountId: AccountId): Promise + } + pointerService: { + flipDefaultWallet(args: { + accountId: AccountId + destinationWalletId: WalletId + }): Promise<{ previousDefaultWalletId: WalletId } | ApplicationError> + } + balanceReader: { + readSourceBalanceUsdCents( + migration: CashWalletMigration, + ): Promise + readDestinationBalanceUsdtMicros( + migration: CashWalletMigration, + ): Promise + } + invoiceService: { + createNoAmountInvoice(args: { + recipientWalletId: WalletId + memo: string + }): Promise + } + paymentService: { + payInvoice(args: { + senderWalletId: WalletId + paymentRequest: string + senderAmountUsdCents?: string + senderAmountUsdtMicros?: string + }): Promise<{ transactionId: IbexTransactionId } | ApplicationError> + } + treasuryService: { + getTreasuryWalletId(): Promise + } +} + +export const isRollbackableCashWalletMigrationStatus = ( + status: CashWalletMigrationStatus, +): boolean => ROLLBACKABLE_STATUSES.includes(status) + +/** + * Pull a migration into rollback (ENG-401). Idempotent at the caller level: + * a migration already in `rollback_started` or `rolled_back` is not an error + * to skip; this function itself enforces the state machine strictly. + * `skipped_already_migrated` is deliberately not rollbackable — those + * accounts were on USDT before the run and are left there. + */ +export const requestCashWalletMigrationRollback = async ({ + migration, + migrationsRepo, + requestedBy, + reason, + requestedAt, +}: { + migration: CashWalletMigration + migrationsRepo: CashWalletMigrationTransitionRepository + requestedBy: string + reason: string + requestedAt: Date +}): Promise => { + const transition = assertCanTransition(migration.status, "rollback_started") + if (transition instanceof Error) return transition + + return migrationsRepo.transitionMigration({ + id: migration.id, + from: migration.status, + to: "rollback_started", + cutoverVersion: migration.cutoverVersion, + runId: migration.runId, + patch: { + rollbackRequestedAt: requestedAt, + rollbackRequestedBy: requestedBy, + rollbackReason: reason, + rollbackFromStatus: migration.status, + }, + }) +} + +const patchRollbackProgress = ({ + migration, + migrationsRepo, + patch, +}: { + migration: CashWalletMigration + migrationsRepo: CashWalletMigrationTransitionRepository + patch: Partial +}) => + migrationsRepo.transitionMigration({ + id: migration.id, + from: "rollback_started", + to: "rollback_started", + cutoverVersion: migration.cutoverVersion, + runId: migration.runId, + patch, + }) + +/** + * Execute (or resume) the rollback of a single migration. Progress is + * field-driven, mirroring the forward pipeline's field-presence guards, so a + * crash at any point resumes idempotently: + * + * 1. restore the default-wallet pointer if it still points at USDT + * 2. reverse the balance move (invoice on the legacy USD wallet, paid from + * the USDT wallet with the exact forward amount) — fails closed if the + * USDT balance no longer covers it + * 3. treasury tops up any remaining USD shortfall vs the original balance + * 4. verify the user is whole (within 1 cent of LN dust) and finalize + * + * Pre-money migrations (no forward payment ever sent) skip 2-3 and + * short-circuit to `rolled_back`. + */ +export const executeCashWalletMigrationRollbackStep = async ({ + migration, + migrationsRepo, + services, +}: { + migration: CashWalletMigration + migrationsRepo: CashWalletMigrationTransitionRepository + services: CashWalletRollbackServices +}): Promise => { + if (migration.status !== "rollback_started") { + return new InvalidCashWalletMigrationTransitionError( + `Rollback step requires status rollback_started, got: ${migration.status}`, + ) + } + + let current = migration + + // 1. Pointer restore — only if the account still defaults to the USDT + // wallet. previousDefaultWalletId (captured at forward pointer flip) is + // preferred; the legacy USD wallet is the correct fallback by construction. + if (current.rollbackPointerRestoredAt === undefined) { + const defaultWalletId = await services.accountReader.getDefaultWalletId( + current.accountId, + ) + if (defaultWalletId instanceof Error) return defaultWalletId + + if (defaultWalletId === current.destinationUsdtWalletId) { + const restored = await services.pointerService.flipDefaultWallet({ + accountId: current.accountId, + destinationWalletId: current.previousDefaultWalletId ?? current.legacyUsdWalletId, + }) + if (restored instanceof Error) return restored + } + + const patched = await patchRollbackProgress({ + migration: current, + migrationsRepo, + patch: { rollbackPointerRestoredAt: services.now() }, + }) + if (patched instanceof Error) return patched + current = patched + } + + const forwardMovedFunds = current.balanceMovePaymentTransactionId !== undefined + + // 2. Reverse balance move. + if (forwardMovedFunds && current.rollbackPaymentTransactionId === undefined) { + if (current.destinationAmountUsdtMicros === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "destinationAmountUsdtMicros is required to reverse a sent balance move", + ) + } + + if (current.destinationAmountUsdtMicros !== "0") { + // Fail closed: if the user transacted on the USDT wallet since the + // forward move, reversing the full amount is not possible without + // operator judgment. + const usdtBalance = + await services.balanceReader.readDestinationBalanceUsdtMicros(current) + if (usdtBalance instanceof Error) return usdtBalance + if (BigInt(usdtBalance) < BigInt(current.destinationAmountUsdtMicros)) { + return new CashWalletMigrationFailedError( + `USDT balance ${usdtBalance} no longer covers reverse amount ${current.destinationAmountUsdtMicros}`, + ) + } + + if ( + current.rollbackInvoicePaymentRequest === undefined || + isInvoicePaymentRequestStale({ + paymentRequest: current.rollbackInvoicePaymentRequest, + now: services.now(), + }) + ) { + const invoice = await services.invoiceService.createNoAmountInvoice({ + recipientWalletId: current.legacyUsdWalletId, + memo: `cwco-rb:${current.runId}:${current.id}:move`, + }) + if (invoice instanceof Error) return invoice + + const patched = await patchRollbackProgress({ + migration: current, + migrationsRepo, + patch: { + rollbackInvoicePaymentRequest: invoice.paymentRequest, + rollbackInvoicePaymentHash: invoice.paymentHash, + }, + }) + if (patched instanceof Error) return patched + current = patched + } + + if (current.rollbackInvoicePaymentRequest === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "rollbackInvoicePaymentRequest is required before reverse payment sending", + ) + } + + const payment = await services.paymentService.payInvoice({ + senderWalletId: current.destinationUsdtWalletId, + paymentRequest: current.rollbackInvoicePaymentRequest, + senderAmountUsdtMicros: current.destinationAmountUsdtMicros, + }) + if (payment instanceof Error) return payment + + const patched = await patchRollbackProgress({ + migration: current, + migrationsRepo, + patch: { rollbackPaymentTransactionId: payment.transactionId }, + }) + if (patched instanceof Error) return patched + current = patched + } + } + + // 3 + 4. Shortfall top-up and verification — only meaningful when funds + // moved forward (and therefore sourceBalanceUsdCents was recorded). + if (forwardMovedFunds) { + if (current.sourceBalanceUsdCents === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "sourceBalanceUsdCents is required to verify a rollback", + ) + } + + const currentUsdCents = + await services.balanceReader.readSourceBalanceUsdCents(current) + if (currentUsdCents instanceof Error) return currentUsdCents + + const shortfall = legacyShortfallUsdtMicros({ + sourceUsdCents: current.sourceBalanceUsdCents, + currentUsdCents, + }) + if (shortfall instanceof Error) return shortfall + + if (BigInt(shortfall) > ROLLBACK_SHORTFALL_TOLERANCE_USDT_MICROS) { + if (current.rollbackShortfallPaymentTransactionId !== undefined) { + // Treasury already topped up once and the user is still short more + // than dust — never double-pay automatically. + return new CashWalletMigrationFailedError( + `Legacy wallet still short ${shortfall} USDT micros after treasury top-up`, + ) + } + + const treasuryWalletId = await services.treasuryService.getTreasuryWalletId() + if (treasuryWalletId instanceof Error) return treasuryWalletId + + if ( + current.rollbackShortfallInvoicePaymentRequest === undefined || + isInvoicePaymentRequestStale({ + paymentRequest: current.rollbackShortfallInvoicePaymentRequest, + now: services.now(), + }) + ) { + const invoice = await services.invoiceService.createNoAmountInvoice({ + recipientWalletId: current.legacyUsdWalletId, + memo: `cwco-rb:${current.runId}:${current.id}:shortfall`, + }) + if (invoice instanceof Error) return invoice + + const patched = await patchRollbackProgress({ + migration: current, + migrationsRepo, + patch: { + rollbackShortfallUsdtMicros: shortfall, + rollbackShortfallInvoicePaymentRequest: invoice.paymentRequest, + rollbackShortfallInvoicePaymentHash: invoice.paymentHash, + }, + }) + if (patched instanceof Error) return patched + current = patched + } + + if (current.rollbackShortfallInvoicePaymentRequest === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "rollbackShortfallInvoicePaymentRequest is required before shortfall payment", + ) + } + + const payment = await services.paymentService.payInvoice({ + senderWalletId: treasuryWalletId, + paymentRequest: current.rollbackShortfallInvoicePaymentRequest, + senderAmountUsdtMicros: shortfall, + }) + if (payment instanceof Error) return payment + + const patched = await patchRollbackProgress({ + migration: current, + migrationsRepo, + patch: { rollbackShortfallPaymentTransactionId: payment.transactionId }, + }) + if (patched instanceof Error) return patched + + // Leave in rollback_started; the next batch pass re-reads the balance + // and finalizes (or fails closed if still short after the top-up). + return patched + } + } + + return migrationsRepo.transitionMigration({ + id: current.id, + from: "rollback_started", + to: "rolled_back", + cutoverVersion: current.cutoverVersion, + runId: current.runId, + patch: { rolledBackAt: services.now() }, + }) +} diff --git a/src/app/cash-wallet-cutover/rollback.ts b/src/app/cash-wallet-cutover/rollback.ts new file mode 100644 index 000000000..26daf658c --- /dev/null +++ b/src/app/cash-wallet-cutover/rollback.ts @@ -0,0 +1,223 @@ +import { CashWalletCutoverRepository } from "@services/mongoose" + +import { + executeCashWalletMigrationRollbackStep, + isRollbackableCashWalletMigrationStatus, + requestCashWalletMigrationRollback, +} from "./rollback-worker" +import { createCashWalletMigrationRuntimeServices } from "./runtime-services" +import { runCashWalletMigrationBatch } from "./runner" +import { ROLLBACKABLE_STATUSES } from "./state-machine" +import { + CashWalletCutoverInProgressError, + CashWalletMigrationFailedError, + InvalidCashWalletCutoverStateTransitionError, +} from "./errors" + +type CashWalletRollbackRepository = ReturnType + +export type CashWalletRollbackRequestReport = { + dryRun: boolean + eligible: number + requested: number + skipped: Partial> + migrationIds: string[] +} + +/** + * Pull migrations into `rollback_started` (ENG-401 admin surface). Idempotent + * and resumable: migrations already in rollback (or rolled back) count as + * skips, never errors, so the request can be re-issued after a partial + * failure. `dryRun` reports what would be requested without writing. + */ +export const requestPrimaryCashWalletRollback = async ({ + cutoverVersion, + runId, + accountId, + reason, + requestedBy, + dryRun = false, + now = new Date(), + migrationsRepo = CashWalletCutoverRepository(), +}: { + cutoverVersion: number + runId: string + /** Single-account mode when set; whole-run mode when omitted. */ + accountId?: AccountId + reason: string + requestedBy: string + dryRun?: boolean + now?: Date + migrationsRepo?: CashWalletRollbackRepository +}): Promise => { + let candidates: CashWalletMigration[] + + if (accountId !== undefined) { + const migration = await migrationsRepo.findMigrationByAccountId({ + accountId, + cutoverVersion, + runId, + }) + if (migration instanceof Error) return migration + if (migration === null) { + return new CashWalletMigrationFailedError( + `No migration found for accountId=${accountId} runId=${runId}`, + ) + } + candidates = [migration] + } else { + const migrations = await migrationsRepo.listMigrationsByStatuses({ + cutoverVersion, + runId, + statuses: [...ROLLBACKABLE_STATUSES, "rollback_started", "rolled_back"], + }) + if (migrations instanceof Error) return migrations + candidates = migrations + } + + const report: CashWalletRollbackRequestReport = { + dryRun, + eligible: 0, + requested: 0, + skipped: {}, + migrationIds: [], + } + + for (const migration of candidates) { + if (!isRollbackableCashWalletMigrationStatus(migration.status)) { + report.skipped[migration.status] = (report.skipped[migration.status] ?? 0) + 1 + continue + } + + report.eligible += 1 + report.migrationIds.push(migration.id) + if (dryRun) continue + + const requested = await requestCashWalletMigrationRollback({ + migration, + migrationsRepo, + requestedBy, + reason, + requestedAt: now, + }) + if (requested instanceof Error) return requested + report.requested += 1 + } + + return report +} + +/** + * Process one locked batch of `rollback_started` migrations. Reuses the + * forward batch runner (locking, stale-lock recovery, failure marking) with + * a rollback-only work list; every failure is marked + * `requires_operator_review` — rollback errors are never retried blindly. + */ +export const runPrimaryCashWalletRollbackBatch = ({ + cutoverVersion, + runId, + workerId, + limit, + stepDelayMs, + lockStaleBefore, + migrationsRepo = CashWalletCutoverRepository(), + runtimeServices = createCashWalletMigrationRuntimeServices(), +}: { + cutoverVersion: number + runId: string + workerId: string + limit?: number + stepDelayMs?: number + lockStaleBefore: Date + migrationsRepo?: CashWalletRollbackRepository + runtimeServices?: ReturnType +}) => + runCashWalletMigrationBatch({ + cutoverVersion, + runId, + workerId, + limit, + stepDelayMs, + lockStaleBefore, + migrationsRepo: { + ...migrationsRepo, + listRunnableMigrations: (args) => + migrationsRepo.listMigrationsByStatuses({ + ...args, + statuses: ["rollback_started"], + }), + markMigrationFailed: (args) => + migrationsRepo.markMigrationFailed({ + ...args, + status: "requires_operator_review", + }), + }, + executor: (migration) => + executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo, + services: runtimeServices, + }), + }) + +/** + * Finalize a run-level rollback: legal only when nothing is still mid-flight + * and nothing failed closed. Flips the cutover config to `rolled_back`; + * `skipped_already_migrated` accounts remain on USDT by design. + */ +export const completePrimaryCashWalletRollback = async ({ + cutoverVersion, + runId, + actor, + now = new Date(), + migrationsRepo = CashWalletCutoverRepository(), +}: { + cutoverVersion: number + runId: string + actor: string + now?: Date + migrationsRepo?: CashWalletRollbackRepository +}): Promise => { + const config = await migrationsRepo.getConfig() + if (config instanceof Error) return config + if (config.state === "rolled_back") return config + if (config.runId !== runId || config.cutoverVersion !== cutoverVersion) { + return new InvalidCashWalletCutoverStateTransitionError( + `Cutover config is tracking runId=${config.runId} cutoverVersion=${config.cutoverVersion}, not the requested rollback run`, + ) + } + + const inFlight = await migrationsRepo.countByStatus({ + cutoverVersion, + runId, + status: "rollback_started", + }) + if (inFlight instanceof Error) return inFlight + if (inFlight > 0) { + return new CashWalletCutoverInProgressError( + `${inFlight} migration(s) still in rollback_started`, + ) + } + + const review = await migrationsRepo.countByStatus({ + cutoverVersion, + runId, + status: "requires_operator_review", + }) + if (review instanceof Error) return review + if (review > 0) { + return new CashWalletMigrationFailedError( + `${review} migration(s) require operator review before completing rollback`, + ) + } + + return migrationsRepo.updateConfig( + { + state: "rolled_back", + cutoverVersion, + runId, + completedAt: now, + }, + actor, + ) +} diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index 1a5d13563..abf75e1f0 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -174,6 +174,17 @@ export const createCashWalletMigrationRuntimeServices = ( return { now: deps.now ?? (() => new Date()), + accountReader: { + // Rollback (ENG-401): read the live default wallet so pointer restore + // is a no-op when the account never flipped (or was already restored). + getDefaultWalletId: async ( + accountId: AccountId, + ): Promise => { + const account = await accountsRepo.findById(accountId) + if (account instanceof Error) return account + return account.defaultWalletId + }, + }, provisioningService: { ensureDestinationWallet: async ({ accountId, diff --git a/src/app/cash-wallet-cutover/state-machine.ts b/src/app/cash-wallet-cutover/state-machine.ts index 9a950e065..b6ff9158e 100644 --- a/src/app/cash-wallet-cutover/state-machine.ts +++ b/src/app/cash-wallet-cutover/state-machine.ts @@ -1,37 +1,92 @@ import { InvalidCashWalletMigrationTransitionError } from "./errors" +// Every status an operator may pull into rollback (ENG-401). Excluded by +// design: `skipped_already_migrated` (those accounts were already on USDT +// before the run and must be left there), `rollback_started` (already in +// rollback), and `rolled_back` (terminal). Pre-money statuses are included so +// a full-run rollback can uniformly cancel them — the rollback handler +// detects that no funds moved and short-circuits straight to `rolled_back`. +export const ROLLBACKABLE_STATUSES: CashWalletMigrationStatus[] = [ + "not_started", + "started", + "provisioned", + "balance_read", + "invoice_created", + "balance_move_sending", + "balance_move_sent", + "balance_move_verified", + "fee_reimbursement_invoice_created", + "fee_reimbursement_sending", + "fee_reimbursed", + "pointer_flipped", + "legacy_zero_verified", + "complete", + "failed", + "requires_operator_review", +] + const transitions: Partial< Record > = { - not_started: ["started"], - started: ["provisioned", "failed"], - provisioned: ["balance_read", "failed", "skipped_already_migrated"], - balance_read: ["invoice_created", "pointer_flipped", "failed"], + not_started: ["started", "rollback_started"], + started: ["provisioned", "failed", "rollback_started"], + provisioned: ["balance_read", "failed", "skipped_already_migrated", "rollback_started"], + balance_read: ["invoice_created", "pointer_flipped", "failed", "rollback_started"], invoice_created: [ "invoice_created", "balance_move_sending", "failed", "requires_operator_review", + "rollback_started", + ], + balance_move_sending: [ + "balance_move_sent", + "failed", + "requires_operator_review", + "rollback_started", + ], + balance_move_sent: [ + "balance_move_verified", + "failed", + "requires_operator_review", + "rollback_started", ], - balance_move_sending: ["balance_move_sent", "failed", "requires_operator_review"], - balance_move_sent: ["balance_move_verified", "failed", "requires_operator_review"], balance_move_verified: [ "fee_reimbursement_invoice_created", "fee_reimbursed", "failed", "requires_operator_review", + "rollback_started", ], fee_reimbursement_invoice_created: [ "fee_reimbursement_invoice_created", "fee_reimbursement_sending", "failed", "requires_operator_review", + "rollback_started", + ], + fee_reimbursement_sending: [ + "fee_reimbursed", + "failed", + "requires_operator_review", + "rollback_started", + ], + fee_reimbursed: ["pointer_flipped", "failed", "rollback_started"], + pointer_flipped: ["legacy_zero_verified", "failed", "rollback_started"], + legacy_zero_verified: ["complete", "failed", "rollback_started"], + complete: ["rollback_started"], + failed: ["rollback_started"], + requires_operator_review: ["rollback_started"], + // Self-loop persists sub-step artifacts (invoice/payment ids) mid-rollback, + // mirroring invoice_created's refresh self-loop. Rollback failures always + // fail closed into operator review — a rollback that dies mid-flight has + // ambiguous side effects by definition. + rollback_started: [ + "rollback_started", + "rolled_back", + "failed", + "requires_operator_review", ], - fee_reimbursement_sending: ["fee_reimbursed", "failed", "requires_operator_review"], - fee_reimbursed: ["pointer_flipped", "failed"], - pointer_flipped: ["legacy_zero_verified", "failed"], - legacy_zero_verified: ["complete", "failed"], - rollback_started: ["rolled_back", "failed"], } export const assertCanTransition = ( diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index b71d0d4f3..2e48e0765 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -91,7 +91,7 @@ const usesNoAmountFeeReimbursementInvoice = ( return BigInt(feeAmountUsdtMicros) < CUTOVER_MIN_FIXED_USDT_INVOICE_MICROS } -const isInvoicePaymentRequestStale = ({ +export const isInvoicePaymentRequestStale = ({ paymentRequest, now, safetyWindowMs = CUTOVER_INVOICE_PAYMENT_SAFETY_WINDOW_MS, diff --git a/src/graphql/admin/mutations.ts b/src/graphql/admin/mutations.ts index fcf3f6ce1..493701d4d 100644 --- a/src/graphql/admin/mutations.ts +++ b/src/graphql/admin/mutations.ts @@ -4,6 +4,7 @@ import AccountUpdateLevelMutation from "@graphql/admin/root/mutation/account-upd import AccountUpdateStatusMutation from "@graphql/admin/root/mutation/account-update-status" import BusinessUpdateMapInfoMutation from "@graphql/admin/root/mutation/business-update-map-info" import CashWalletCutoverUpdateMutation from "@graphql/admin/root/mutation/cash-wallet-cutover-update" +import CashWalletCutoverRollbackMutation from "@graphql/admin/root/mutation/cash-wallet-cutover-rollback" import UserUpdatePhoneMutation from "./root/mutation/user-update-phone" import BusinessDeleteMapInfoMutation from "./root/mutation/delete-business-map" @@ -26,6 +27,7 @@ export const mutationFields = { sendNotification: SendNotificationMutation, cashoutNotificationSend: sendCashoutSettledNotification, cashWalletCutoverUpdate: CashWalletCutoverUpdateMutation, + cashWalletCutoverRollback: CashWalletCutoverRollbackMutation, }, } diff --git a/src/graphql/admin/queries.ts b/src/graphql/admin/queries.ts index deafdfd9f..5d28b7f8b 100644 --- a/src/graphql/admin/queries.ts +++ b/src/graphql/admin/queries.ts @@ -17,6 +17,7 @@ import MerchantsPendingApprovalQuery from "./root/query/merchants-pending-approv import IdDocumentReadUrlQuery from "./root/query/id-document-read-url" import NotificationTopicsQuery from "./root/query/notification-topics" import BridgeReconciliationOrphansQuery from "./root/query/bridge-reconciliation-orphans" +import CashWalletMigrationsQuery from "./root/query/cash-wallet-migrations" export const queryFields = { unauthed: {}, @@ -38,6 +39,7 @@ export const queryFields = { notificationTopics: NotificationTopicsQuery, bridgeReconciliationOrphans: BridgeReconciliationOrphansQuery, cashWalletCutover: CashWalletCutoverQuery, + cashWalletMigrations: CashWalletMigrationsQuery, }, } diff --git a/src/graphql/admin/root/mutation/cash-wallet-cutover-rollback.ts b/src/graphql/admin/root/mutation/cash-wallet-cutover-rollback.ts new file mode 100644 index 000000000..a0e010da2 --- /dev/null +++ b/src/graphql/admin/root/mutation/cash-wallet-cutover-rollback.ts @@ -0,0 +1,79 @@ +import { GT } from "@graphql/index" +import { mapAndParseErrorForGqlResponse } from "@graphql/error-map" +import IError from "@graphql/shared/types/abstract/error" +import { requestPrimaryCashWalletRollback } from "@app/cash-wallet-cutover" + +const CashWalletCutoverRollbackInput = GT.Input({ + name: "CashWalletCutoverRollbackInput", + fields: () => ({ + cutoverVersion: { type: GT.NonNull(GT.Int) }, + runId: { type: GT.NonNull(GT.String) }, + // Single-account rollback when set; whole-run rollback when omitted. + accountId: { type: GT.String }, + reason: { type: GT.NonNull(GT.String) }, + dryRun: { type: GT.Boolean }, + }), +}) + +const CashWalletCutoverRollbackPayload = GT.Object({ + name: "CashWalletCutoverRollbackPayload", + fields: () => ({ + errors: { type: GT.NonNullList(IError) }, + dryRun: { type: GT.Boolean }, + eligible: { type: GT.Int }, + requested: { type: GT.Int }, + migrationIds: { type: GT.List(GT.String) }, + skippedByStatus: { type: GT.List(GT.String) }, + }), +}) + +// ENG-401 #6: admin-only rollback request. Marks eligible migrations +// `rollback_started` (single-account or full-run); execution happens in +// locked worker batches (CLI `rollback-batch` / runPrimaryCashWalletRollbackBatch), +// so this mutation is fast, idempotent (already-rolled-back records are +// skips), and resumable after partial failures. `skipped_already_migrated` +// accounts are never eligible — they stay on USDT. +const CashWalletCutoverRollbackMutation = GT.Field< + null, + GraphQLAdminContext, + { + input: { + cutoverVersion: number + runId: string + accountId?: string + reason: string + dryRun?: boolean + } + } +>({ + type: GT.NonNull(CashWalletCutoverRollbackPayload), + args: { + input: { type: GT.NonNull(CashWalletCutoverRollbackInput) }, + }, + resolve: async (_, { input }, ctx) => { + const report = await requestPrimaryCashWalletRollback({ + cutoverVersion: input.cutoverVersion, + runId: input.runId, + accountId: input.accountId as AccountId | undefined, + reason: input.reason, + requestedBy: ctx.user.id, + dryRun: input.dryRun ?? false, + }) + if (report instanceof Error) { + return { errors: [mapAndParseErrorForGqlResponse(report)] } + } + + return { + errors: [], + dryRun: report.dryRun, + eligible: report.eligible, + requested: report.requested, + migrationIds: report.migrationIds, + skippedByStatus: Object.entries(report.skipped).map( + ([status, count]) => `${status}:${count}`, + ), + } + }, +}) + +export default CashWalletCutoverRollbackMutation diff --git a/src/graphql/admin/root/query/cash-wallet-migrations.ts b/src/graphql/admin/root/query/cash-wallet-migrations.ts new file mode 100644 index 000000000..f23c0b0f4 --- /dev/null +++ b/src/graphql/admin/root/query/cash-wallet-migrations.ts @@ -0,0 +1,59 @@ +import { GT } from "@graphql/index" +import CashWalletMigrationObject from "@graphql/admin/types/object/cash-wallet-migration" +import { CashWalletCutoverRepository } from "@services/mongoose/cash-wallet-cutover" + +// ENG-401: operator inspection of migration records (incl. +// previousDefaultWalletId and rollback progress) without direct DB access. +const CashWalletMigrationsQuery = GT.Field< + null, + GraphQLAdminContext, + { + runId: string + cutoverVersion: number + statuses?: string[] + } +>({ + type: GT.NonNullList(CashWalletMigrationObject), + args: { + runId: { type: GT.NonNull(GT.String) }, + cutoverVersion: { type: GT.NonNull(GT.Int) }, + statuses: { type: GT.List(GT.String) }, + }, + resolve: async (_, { runId, cutoverVersion, statuses }) => { + const repo = CashWalletCutoverRepository() + + const migrations = await repo.listMigrationsByStatuses({ + cutoverVersion, + runId, + statuses: + statuses && statuses.length > 0 + ? (statuses as CashWalletMigrationStatus[]) + : ([ + "not_started", + "started", + "provisioned", + "balance_read", + "invoice_created", + "balance_move_sending", + "balance_move_sent", + "balance_move_verified", + "fee_reimbursement_invoice_created", + "fee_reimbursement_sending", + "fee_reimbursed", + "pointer_flipped", + "legacy_zero_verified", + "complete", + "failed", + "requires_operator_review", + "skipped_already_migrated", + "rollback_started", + "rolled_back", + ] as CashWalletMigrationStatus[]), + }) + if (migrations instanceof Error) throw migrations + + return migrations + }, +}) + +export default CashWalletMigrationsQuery diff --git a/src/graphql/admin/schema.graphql b/src/graphql/admin/schema.graphql index 46da132ac..fb9d55404 100644 --- a/src/graphql/admin/schema.graphql +++ b/src/graphql/admin/schema.graphql @@ -154,10 +154,28 @@ type CashWalletCutoverPayload { errors: [Error!]! } +input CashWalletCutoverRollbackInput { + accountId: String + cutoverVersion: Int! + dryRun: Boolean + reason: String! + runId: String! +} + +type CashWalletCutoverRollbackPayload { + dryRun: Boolean + eligible: Int + errors: [Error!]! + migrationIds: [String] + requested: Int + skippedByStatus: [String] +} + enum CashWalletCutoverState { COMPLETE IN_PROGRESS PRE + ROLLED_BACK } input CashWalletCutoverUpdateInput { @@ -168,6 +186,38 @@ input CashWalletCutoverUpdateInput { state: CashWalletCutoverState! } +type CashWalletMigration { + accountId: String! + attempts: Int! + balanceMovePaymentTransactionId: String + completedAt: Timestamp + cutoverVersion: Int! + destinationAmountUsdtMicros: String + destinationStartingBalanceUsdtMicros: String + destinationUsdtWalletId: String! + feeAmountUsdCents: String + feeAmountUsdtMicros: String + feeReimbursementPaymentTransactionId: String + id: ID! + lastError: String + legacyUsdWalletId: String! + previousDefaultWalletId: String + rollbackFromStatus: String + rollbackPaymentTransactionId: String + rollbackPointerRestoredAt: Timestamp + rollbackReason: String + rollbackRequestedAt: Timestamp + rollbackRequestedBy: String + rollbackShortfallPaymentTransactionId: String + rollbackShortfallUsdtMicros: String + rolledBackAt: Timestamp + runId: String! + sourceBalanceUsdCents: String + startedAt: Timestamp + status: String! + updatedAt: Timestamp! +} + input CashoutNotificationSendInput { accountId: String! amount: Int! @@ -310,6 +360,7 @@ type Mutation { accountUpdateStatus(input: AccountUpdateStatusInput!): AccountDetailPayload! businessDeleteMapInfo(input: BusinessDeleteMapInfoInput!): AccountDetailPayload! businessUpdateMapInfo(input: BusinessUpdateMapInfoInput!): AccountDetailPayload! + cashWalletCutoverRollback(input: CashWalletCutoverRollbackInput!): CashWalletCutoverRollbackPayload! cashWalletCutoverUpdate(input: CashWalletCutoverUpdateInput!): CashWalletCutoverPayload! cashoutNotificationSend(input: CashoutNotificationSendInput!): SuccessPayload! merchantMapDelete(input: MerchantMapDeleteInput!): MerchantPayload! @@ -369,6 +420,7 @@ type Query { allLevels: [AccountLevel!]! bridgeReconciliationOrphans(limit: Int = 50, orphanType: String = null, status: String = null): [BridgeReconciliationOrphan!]! cashWalletCutover: CashWalletCutover! + cashWalletMigrations(cutoverVersion: Int!, runId: String!, statuses: [String]): [CashWalletMigration!]! idDocumentReadUrl( """Storage key of the ID document file""" fileKey: String! diff --git a/src/graphql/admin/types/object/cash-wallet-migration.ts b/src/graphql/admin/types/object/cash-wallet-migration.ts new file mode 100644 index 000000000..3145157fe --- /dev/null +++ b/src/graphql/admin/types/object/cash-wallet-migration.ts @@ -0,0 +1,44 @@ +import { GT } from "@graphql/index" +import Timestamp from "@graphql/shared/types/scalar/timestamp" + +// Admin-only view of a per-account cutover migration record (ENG-401). +// Exposes previousDefaultWalletId and the full rollback audit trail so +// operators never need direct DB access during an incident. The status +// fields are plain strings on purpose: the internal status set evolves with +// the state machine and this surface must never lag it. +const CashWalletMigrationObject = GT.Object({ + name: "CashWalletMigration", + fields: () => ({ + id: { type: GT.NonNullID }, + accountId: { type: GT.NonNull(GT.String) }, + status: { type: GT.NonNull(GT.String) }, + runId: { type: GT.NonNull(GT.String) }, + cutoverVersion: { type: GT.NonNull(GT.Int) }, + legacyUsdWalletId: { type: GT.NonNull(GT.String) }, + destinationUsdtWalletId: { type: GT.NonNull(GT.String) }, + previousDefaultWalletId: { type: GT.String }, + sourceBalanceUsdCents: { type: GT.String }, + destinationAmountUsdtMicros: { type: GT.String }, + destinationStartingBalanceUsdtMicros: { type: GT.String }, + feeAmountUsdCents: { type: GT.String }, + feeAmountUsdtMicros: { type: GT.String }, + balanceMovePaymentTransactionId: { type: GT.String }, + feeReimbursementPaymentTransactionId: { type: GT.String }, + attempts: { type: GT.NonNull(GT.Int) }, + lastError: { type: GT.String }, + startedAt: { type: Timestamp }, + completedAt: { type: Timestamp }, + updatedAt: { type: GT.NonNull(Timestamp) }, + rollbackRequestedAt: { type: Timestamp }, + rollbackRequestedBy: { type: GT.String }, + rollbackReason: { type: GT.String }, + rollbackFromStatus: { type: GT.String }, + rollbackPointerRestoredAt: { type: Timestamp }, + rollbackPaymentTransactionId: { type: GT.String }, + rollbackShortfallUsdtMicros: { type: GT.String }, + rollbackShortfallPaymentTransactionId: { type: GT.String }, + rolledBackAt: { type: Timestamp }, + }), +}) + +export default CashWalletMigrationObject diff --git a/src/graphql/public/schema.graphql b/src/graphql/public/schema.graphql index ce0a2a4fe..76ec75b16 100644 --- a/src/graphql/public/schema.graphql +++ b/src/graphql/public/schema.graphql @@ -449,6 +449,7 @@ enum CashWalletCutoverState { COMPLETE IN_PROGRESS PRE + ROLLED_BACK } type CashoutOffer { diff --git a/src/graphql/shared/types/scalar/cash-wallet-cutover-state.ts b/src/graphql/shared/types/scalar/cash-wallet-cutover-state.ts index 831a377f8..2d66493e9 100644 --- a/src/graphql/shared/types/scalar/cash-wallet-cutover-state.ts +++ b/src/graphql/shared/types/scalar/cash-wallet-cutover-state.ts @@ -6,6 +6,9 @@ const CashWalletCutoverState = GT.Enum({ PRE: { value: "pre" }, IN_PROGRESS: { value: "in_progress" }, COMPLETE: { value: "complete" }, + // Terminal state for a run reversed via the ENG-401 rollback path; a new + // run (fresh runId) may be prepared and started afterwards. + ROLLED_BACK: { value: "rolled_back" }, }, }) diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 5e481a074..1d5762f52 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -24,6 +24,15 @@ const args = yargs(hideBin(process.argv)) .command("run-batch", "run one locked migration worker batch") .command("status", "print cutover config and migration counts") .command("complete", "mark cutover complete after all migrations finish") + .command( + "rollback-request", + "pull eligible migrations into rollback_started (single account with --account-id, else whole run)", + ) + .command("rollback-batch", "run one locked rollback worker batch") + .command( + "rollback-complete", + "mark the cutover config rolled_back once no migrations remain in rollback_started", + ) .demandCommand(1) .option("cutover-version", { type: "number", demandOption: true }) .option("run-id", { type: "string", demandOption: true }) @@ -36,6 +45,8 @@ const args = yargs(hideBin(process.argv)) .option("provision-retry-delay-ms", { type: "number", default: 60_000 }) .option("max-provision-attempts", { type: "number", default: 5 }) .option("dry-run", { type: "boolean", default: false }) + .option("account-id", { type: "string" }) + .option("reason", { type: "string", default: "" }) .option("lock-stale-seconds", { type: "number", default: 300 }) .option("configPath", { type: "string", demandOption: true }) .parseSync() @@ -149,6 +160,51 @@ const run = async () => { return } + case "rollback-request": { + if (!args.reason) { + throw new Error("--reason is required for rollback-request") + } + const result = await CashWalletCutover.requestPrimaryCashWalletRollback({ + cutoverVersion, + runId, + accountId: args["account-id"] as AccountId | undefined, + reason: args.reason, + requestedBy: args.operator, + dryRun: args["dry-run"], + migrationsRepo: repository, + }) + if (result instanceof Error) throw result + toJson(result) + return + } + + case "rollback-batch": { + const result = await CashWalletCutover.runPrimaryCashWalletRollbackBatch({ + cutoverVersion, + runId, + workerId: args["worker-id"], + limit: args.limit, + stepDelayMs: args["step-delay-ms"], + lockStaleBefore: new Date(Date.now() - args["lock-stale-seconds"] * 1000), + migrationsRepo: repository, + }) + if (result instanceof Error) throw result + toJson(result) + return + } + + case "rollback-complete": { + const result = await CashWalletCutover.completePrimaryCashWalletRollback({ + cutoverVersion, + runId, + actor: args.operator, + migrationsRepo: repository, + }) + if (result instanceof Error) throw result + toJson(result) + return + } + default: throw new Error(`Unsupported cash wallet cutover command: ${command}`) } diff --git a/src/services/mongoose/cash-wallet-cutover.ts b/src/services/mongoose/cash-wallet-cutover.ts index f5c9e8da0..d0ce9f4c7 100644 --- a/src/services/mongoose/cash-wallet-cutover.ts +++ b/src/services/mongoose/cash-wallet-cutover.ts @@ -100,6 +100,19 @@ const resultToMigration = (record: CashWalletMigrationRecord): CashWalletMigrati startedAt: record.startedAt, completedAt: record.completedAt, updatedAt: record.updatedAt, + rollbackRequestedAt: record.rollbackRequestedAt, + rollbackRequestedBy: record.rollbackRequestedBy, + rollbackReason: record.rollbackReason, + rollbackFromStatus: record.rollbackFromStatus, + rollbackPointerRestoredAt: record.rollbackPointerRestoredAt, + rollbackInvoicePaymentRequest: record.rollbackInvoicePaymentRequest, + rollbackInvoicePaymentHash: record.rollbackInvoicePaymentHash, + rollbackPaymentTransactionId: record.rollbackPaymentTransactionId, + rollbackShortfallUsdtMicros: record.rollbackShortfallUsdtMicros, + rollbackShortfallInvoicePaymentRequest: record.rollbackShortfallInvoicePaymentRequest, + rollbackShortfallInvoicePaymentHash: record.rollbackShortfallInvoicePaymentHash, + rollbackShortfallPaymentTransactionId: record.rollbackShortfallPaymentTransactionId, + rolledBackAt: record.rolledBackAt, }) export const CashWalletCutoverRepository = () => { @@ -312,6 +325,34 @@ export const CashWalletCutoverRepository = () => { } } + // ENG-401: work list for rollback — powers "everything eligible for + // rollback in run X" (batch request) and "everything mid-rollback" (batch + // execution). Status-set driven so callers own the semantics. + const listMigrationsByStatuses = async ({ + cutoverVersion, + runId, + statuses, + limit, + }: { + cutoverVersion: number + runId: string + statuses: CashWalletMigrationStatus[] + limit?: number + }): Promise => { + try { + const results = await CashWalletMigration.find({ + cutoverVersion, + runId, + status: { $in: statuses }, + }) + .sort({ updatedAt: 1 }) + .limit(limit ?? 0) + return results.map(resultToMigration) + } catch (err) { + return parseRepositoryError(err) + } + } + const countByStatus = async ({ cutoverVersion, runId, @@ -338,6 +379,7 @@ export const CashWalletCutoverRepository = () => { releaseMigrationLock, markMigrationFailed, listRunnableMigrations, + listMigrationsByStatuses, countByStatus, } } diff --git a/src/services/mongoose/schema.ts b/src/services/mongoose/schema.ts index 9e8835531..2ac17b080 100644 --- a/src/services/mongoose/schema.ts +++ b/src/services/mongoose/schema.ts @@ -702,6 +702,19 @@ const CashWalletMigrationSchema = new Schema({ startedAt: Date, completedAt: Date, updatedAt: { type: Date, default: Date.now, index: true }, + rollbackRequestedAt: Date, + rollbackRequestedBy: String, + rollbackReason: String, + rollbackFromStatus: String, + rollbackPointerRestoredAt: Date, + rollbackInvoicePaymentRequest: String, + rollbackInvoicePaymentHash: String, + rollbackPaymentTransactionId: String, + rollbackShortfallUsdtMicros: String, + rollbackShortfallInvoicePaymentRequest: String, + rollbackShortfallInvoicePaymentHash: String, + rollbackShortfallPaymentTransactionId: String, + rolledBackAt: Date, }) CashWalletMigrationSchema.index({ accountId: 1, runId: 1 }, { unique: true }) diff --git a/src/services/mongoose/schema.types.d.ts b/src/services/mongoose/schema.types.d.ts index 332c918f5..43c4e3027 100644 --- a/src/services/mongoose/schema.types.d.ts +++ b/src/services/mongoose/schema.types.d.ts @@ -147,6 +147,19 @@ interface CashWalletMigrationRecord { startedAt?: Date completedAt?: Date updatedAt: Date + rollbackRequestedAt?: Date + rollbackRequestedBy?: string + rollbackReason?: string + rollbackFromStatus?: CashWalletMigrationStatus + rollbackPointerRestoredAt?: Date + rollbackInvoicePaymentRequest?: string + rollbackInvoicePaymentHash?: string + rollbackPaymentTransactionId?: string + rollbackShortfallUsdtMicros?: string + rollbackShortfallInvoicePaymentRequest?: string + rollbackShortfallInvoicePaymentHash?: string + rollbackShortfallPaymentTransactionId?: string + rolledBackAt?: Date } interface LocationRecord { diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts new file mode 100644 index 000000000..c3cedfedf --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts @@ -0,0 +1,351 @@ +import { + executeCashWalletMigrationRollbackStep, + requestCashWalletMigrationRollback, +} from "@app/cash-wallet-cutover/rollback-worker" +import { legacyShortfallUsdtMicros } from "@app/cash-wallet-cutover/amount-conversion" +import { assertCanTransition } from "@app/cash-wallet-cutover/state-machine" +import { + CashWalletMigrationFailedError, + InvalidCashWalletMigrationTransitionError, +} from "@app/cash-wallet-cutover/errors" + +const NOW = new Date("2026-07-04T00:00:00.000Z") + +const baseMigration = { + id: "migration-id", + accountId: "account-id" as AccountId, + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, + cutoverVersion: 3, + runId: "run-id", + status: "rollback_started", + idempotencyKey: "run-id:account-id", + attempts: 0, + updatedAt: NOW, +} as CashWalletMigration + +const invoice = { + paymentRequest: "lnbc1invoice" as Bolt11, + paymentHash: "payment-hash" as PaymentHash, +} + +// Stateful repo mock: patches accumulate across the executor's sub-steps the +// way the real repository behaves. +const statefulRepo = (initial: CashWalletMigration) => { + let state = { ...initial } + return { + transitionMigration: jest.fn(async ({ to, patch }) => { + state = { ...state, ...patch, status: to } + return { ...state } + }), + current: () => state, + } +} + +const services = ({ + defaultWalletId = "legacy-wallet-id" as WalletId, + usdtBalanceMicros = "0", + legacyBalanceUsdCents = "0", +}: { + defaultWalletId?: WalletId + usdtBalanceMicros?: string + legacyBalanceUsdCents?: string +} = {}) => ({ + now: () => NOW, + accountReader: { + getDefaultWalletId: jest.fn().mockResolvedValue(defaultWalletId), + }, + pointerService: { + flipDefaultWallet: jest + .fn() + .mockResolvedValue({ previousDefaultWalletId: "destination-wallet-id" }), + }, + balanceReader: { + readSourceBalanceUsdCents: jest.fn().mockResolvedValue(legacyBalanceUsdCents), + readDestinationBalanceUsdtMicros: jest.fn().mockResolvedValue(usdtBalanceMicros), + }, + invoiceService: { + createNoAmountInvoice: jest.fn().mockResolvedValue(invoice), + }, + paymentService: { + payInvoice: jest + .fn() + .mockResolvedValue({ transactionId: "reverse-txn-id" as IbexTransactionId }), + }, + treasuryService: { + getTreasuryWalletId: jest.fn().mockResolvedValue("treasury-wallet-id" as WalletId), + }, +}) + +describe("rollback request", () => { + it("pulls a completed migration into rollback_started with an audit trail", async () => { + const migration = { ...baseMigration, status: "complete" } as CashWalletMigration + const migrationsRepo = statefulRepo(migration) + + const result = await requestCashWalletMigrationRollback({ + migration, + migrationsRepo, + requestedBy: "operator-1", + reason: "reconciliation drift", + requestedAt: NOW, + }) + + expect(result).toMatchObject({ + status: "rollback_started", + rollbackRequestedAt: NOW, + rollbackRequestedBy: "operator-1", + rollbackReason: "reconciliation drift", + rollbackFromStatus: "complete", + }) + }) + + it("rejects rollback of skipped_already_migrated accounts", async () => { + const migration = { + ...baseMigration, + status: "skipped_already_migrated", + } as CashWalletMigration + + const result = await requestCashWalletMigrationRollback({ + migration, + migrationsRepo: statefulRepo(migration), + requestedBy: "operator-1", + reason: "should not happen", + requestedAt: NOW, + }) + + expect(result).toBeInstanceOf(InvalidCashWalletMigrationTransitionError) + }) + + it("rejects rollback of already rolled_back migrations", () => { + expect(assertCanTransition("rolled_back", "rollback_started")).toBeInstanceOf( + InvalidCashWalletMigrationTransitionError, + ) + }) +}) + +describe("rollback executor", () => { + it("refuses to run on non-rollback_started migrations", async () => { + const result = await executeCashWalletMigrationRollbackStep({ + migration: { ...baseMigration, status: "complete" } as CashWalletMigration, + migrationsRepo: statefulRepo(baseMigration), + services: services(), + }) + expect(result).toBeInstanceOf(InvalidCashWalletMigrationTransitionError) + }) + + it("short-circuits pre-money migrations straight to rolled_back", async () => { + // Never sent a forward payment; account default never flipped. + const migration = { + ...baseMigration, + rollbackFromStatus: "balance_read", + } as CashWalletMigration + const migrationsRepo = statefulRepo(migration) + const svc = services() + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo, + services: svc, + }) + + expect(result).toMatchObject({ status: "rolled_back", rolledBackAt: NOW }) + expect(svc.pointerService.flipDefaultWallet).not.toHaveBeenCalled() + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + }) + + it("restores the default pointer when the account still defaults to USDT", async () => { + const migration = { + ...baseMigration, + previousDefaultWalletId: "legacy-wallet-id" as WalletId, + } + const migrationsRepo = statefulRepo(migration) + const svc = services({ defaultWalletId: "destination-wallet-id" as WalletId }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo, + services: svc, + }) + + expect(svc.pointerService.flipDefaultWallet).toHaveBeenCalledWith({ + accountId: migration.accountId, + destinationWalletId: "legacy-wallet-id", + }) + expect(result).toMatchObject({ status: "rolled_back" }) + }) + + it("reverses the balance move with the exact forward amount", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + } + const migrationsRepo = statefulRepo(migration) + const svc = services({ + usdtBalanceMicros: "1500000000", + legacyBalanceUsdCents: "150000", // whole again after the reverse pay + }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo, + services: svc, + }) + + expect(svc.invoiceService.createNoAmountInvoice).toHaveBeenCalledWith({ + recipientWalletId: migration.legacyUsdWalletId, + memo: `cwco-rb:${migration.runId}:${migration.id}:move`, + }) + expect(svc.paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: migration.destinationUsdtWalletId, + paymentRequest: invoice.paymentRequest, + senderAmountUsdtMicros: "1500000000", + }) + expect(result).toMatchObject({ + status: "rolled_back", + rollbackPaymentTransactionId: "reverse-txn-id", + }) + }) + + it("fails closed when the USDT balance no longer covers the reverse amount", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + } + const svc = services({ usdtBalanceMicros: "900000000" }) // user spent funds + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(result).toBeInstanceOf(CashWalletMigrationFailedError) + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + }) + + it("resumes after a completed reverse payment without paying again", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + rollbackPointerRestoredAt: NOW, + rollbackPaymentTransactionId: "reverse-txn-id", + } + const svc = services({ legacyBalanceUsdCents: "150000" }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + expect(result).toMatchObject({ status: "rolled_back" }) + }) + + it("tops up a legacy shortfall from treasury and stays in rollback_started", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + rollbackPointerRestoredAt: NOW, + rollbackPaymentTransactionId: "reverse-txn-id", + } + const migrationsRepo = statefulRepo(migration) + // 149990 cents received: 10 cents short = 100_000 micros > 1-cent tolerance + const svc = services({ legacyBalanceUsdCents: "149990" }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo, + services: svc, + }) + + expect(svc.invoiceService.createNoAmountInvoice).toHaveBeenCalledWith({ + recipientWalletId: migration.legacyUsdWalletId, + memo: `cwco-rb:${migration.runId}:${migration.id}:shortfall`, + }) + expect(svc.paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: "treasury-wallet-id", + paymentRequest: invoice.paymentRequest, + senderAmountUsdtMicros: "100000", + }) + expect(result).toMatchObject({ + status: "rollback_started", + rollbackShortfallPaymentTransactionId: "reverse-txn-id", + }) + }) + + it("tolerates sub-cent dust and finalizes without a treasury payment", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + rollbackPointerRestoredAt: NOW, + rollbackPaymentTransactionId: "reverse-txn-id", + } + // 0.5 cents short = 5_000 micros <= tolerance + const svc = services({ legacyBalanceUsdCents: "149999.5" }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + expect(result).toMatchObject({ status: "rolled_back" }) + }) + + it("never double-pays: still short after a treasury top-up fails closed", async () => { + const migration = { + ...baseMigration, + balanceMovePaymentTransactionId: "forward-txn-id", + sourceBalanceUsdCents: "150000", + destinationAmountUsdtMicros: "1500000000", + rollbackPointerRestoredAt: NOW, + rollbackPaymentTransactionId: "reverse-txn-id", + rollbackShortfallPaymentTransactionId: "shortfall-txn-id", + } + const svc = services({ legacyBalanceUsdCents: "149000" }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(result).toBeInstanceOf(CashWalletMigrationFailedError) + expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() + }) +}) + +describe("legacyShortfallUsdtMicros", () => { + it("returns zero when the wallet is whole", () => { + expect( + legacyShortfallUsdtMicros({ sourceUsdCents: "150000", currentUsdCents: "150000" }), + ).toEqual("0") + }) + + it("returns zero when the wallet holds more than the source", () => { + expect( + legacyShortfallUsdtMicros({ sourceUsdCents: "150000", currentUsdCents: "150001" }), + ).toEqual("0") + }) + + it("computes decimal-cent shortfalls in micros", () => { + expect( + legacyShortfallUsdtMicros({ + sourceUsdCents: "150000.25", + currentUsdCents: "149999.5", + }), + ).toEqual("7500") + }) +}) From c19afb9604a65ec85d067fa6abad2010f356b105 Mon Sep 17 00:00:00 2001 From: Dread Date: Sat, 4 Jul 2026 12:57:06 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(cutover):=20address=20rollback=20review?= =?UTF-8?q?=20=E2=80=94=20completion=20gating,=20guard=20routing,=20pointe?= =?UTF-8?q?r-restore=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes on the ENG-401 rollback implementation: 1. (High) completePrimaryCashWalletRollback could flip the run-level config to rolled_back after a single-account rollback while every other migration remained `complete`. Completion now requires the ENTIRE run to be accounted for: zero migrations outside {rolled_back, skipped_already_migrated} — checked via listMigrationsByStatuses over every other status, with the offending migration named in the error. Single-account rollbacks never complete the run. 2. (High) evaluateCashWalletCutoverGuard now handles the rolled_back config state explicitly instead of falling through to the in_progress branch: rolled_back/absent/not_started -> legacy_usd, skipped_already_migrated -> usdt (they were USDT before the run), rollback_started -> in-progress error, and anything else (e.g. a leftover `complete` under a supposedly rolled-back run) fails closed with CashWalletMigrationFailedError rather than guessing a route. 3. (Bonus, same class as 2) the `complete` config state blanket-routed every account to usdt — including accounts rolled back AFTER global complete (ENG-364 mode 3D), whose funds are back on legacy USD. The complete branch now routes rolled_back -> legacy_usd and blocks rollback_started, and only then blanket-routes usdt. 4. (Medium) pointer restore is now gated on previousDefaultWalletId being present — the forward pointer flip is that field's only writer, so its absence proves no flip happened. A pre-money migration whose account defaults to USDT (e.g. native USDT-default signup) is no longer touched; the fallback flip to legacyUsdWalletId is removed. 9 new unit tests (rollback.spec.ts + rollback-worker.spec.ts): all 8 cutover suites pass, 44 total. tsc + eslint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/app/cash-wallet-cutover/guard.ts | 31 +++- .../cash-wallet-cutover/rollback-worker.ts | 31 ++-- src/app/cash-wallet-cutover/rollback.ts | 46 ++++-- .../rollback-worker.spec.ts | 16 ++ .../app/cash-wallet-cutover/rollback.spec.ts | 153 ++++++++++++++++++ 5 files changed, 246 insertions(+), 31 deletions(-) create mode 100644 test/flash/unit/app/cash-wallet-cutover/rollback.spec.ts diff --git a/src/app/cash-wallet-cutover/guard.ts b/src/app/cash-wallet-cutover/guard.ts index 214968d4c..f52c7dfa2 100644 --- a/src/app/cash-wallet-cutover/guard.ts +++ b/src/app/cash-wallet-cutover/guard.ts @@ -36,7 +36,36 @@ export const evaluateCashWalletCutoverGuard = ({ migration?: CashWalletMigration | null }): { route: CashWalletCutoverRoute } | ApplicationError => { if (cutover.state === "pre") return { route: "legacy_usd" } - if (cutover.state === "complete") return { route: "usdt" } + + if (cutover.state === "complete") { + // Single-account rollback can occur after global complete (ENG-364 mode + // 3D): those accounts' funds are back on the legacy USD wallet and must + // not be blanket-routed to USDT. + if (migration?.status === "rollback_started") { + return new CashWalletCutoverInProgressError() + } + if (migration?.status === "rolled_back") return { route: "legacy_usd" } + return { route: "usdt" } + } + + if (cutover.state === "rolled_back") { + // Run-level rollback completion requires every migration to be + // rolled_back or skipped_already_migrated (completePrimaryCashWalletRollback + // enforces this), so route by where funds actually sit and fail closed + // on anything inconsistent with a rolled-back run. + if ( + !migration || + migration.status === "not_started" || + migration.status === "rolled_back" + ) { + return { route: "legacy_usd" } + } + if (migration.status === "skipped_already_migrated") return { route: "usdt" } + if (migration.status === "rollback_started") { + return new CashWalletCutoverInProgressError() + } + return new CashWalletMigrationFailedError() + } if (!migration || migration.status === "not_started") return { route: "legacy_usd" } if ( diff --git a/src/app/cash-wallet-cutover/rollback-worker.ts b/src/app/cash-wallet-cutover/rollback-worker.ts index 9587ab814..03046e721 100644 --- a/src/app/cash-wallet-cutover/rollback-worker.ts +++ b/src/app/cash-wallet-cutover/rollback-worker.ts @@ -152,21 +152,26 @@ export const executeCashWalletMigrationRollbackStep = async ({ let current = migration - // 1. Pointer restore — only if the account still defaults to the USDT - // wallet. previousDefaultWalletId (captured at forward pointer flip) is - // preferred; the legacy USD wallet is the correct fallback by construction. + // 1. Pointer restore — only when the forward pipeline actually flipped the + // pointer, evidenced by previousDefaultWalletId (the flip is the only + // writer of that field). Pre-money migrations never flipped; if such an + // account defaults to USDT it got there by other means (e.g. native + // USDT-default signup) and the rollback must not touch it. The live + // default is still checked so an already-restored pointer is a no-op. if (current.rollbackPointerRestoredAt === undefined) { - const defaultWalletId = await services.accountReader.getDefaultWalletId( - current.accountId, - ) - if (defaultWalletId instanceof Error) return defaultWalletId + if (current.previousDefaultWalletId !== undefined) { + const defaultWalletId = await services.accountReader.getDefaultWalletId( + current.accountId, + ) + if (defaultWalletId instanceof Error) return defaultWalletId - if (defaultWalletId === current.destinationUsdtWalletId) { - const restored = await services.pointerService.flipDefaultWallet({ - accountId: current.accountId, - destinationWalletId: current.previousDefaultWalletId ?? current.legacyUsdWalletId, - }) - if (restored instanceof Error) return restored + if (defaultWalletId === current.destinationUsdtWalletId) { + const restored = await services.pointerService.flipDefaultWallet({ + accountId: current.accountId, + destinationWalletId: current.previousDefaultWalletId, + }) + if (restored instanceof Error) return restored + } } const patched = await patchRollbackProgress({ diff --git a/src/app/cash-wallet-cutover/rollback.ts b/src/app/cash-wallet-cutover/rollback.ts index 26daf658c..445119d1e 100644 --- a/src/app/cash-wallet-cutover/rollback.ts +++ b/src/app/cash-wallet-cutover/rollback.ts @@ -187,27 +187,39 @@ export const completePrimaryCashWalletRollback = async ({ ) } - const inFlight = await migrationsRepo.countByStatus({ + // Run-level completion requires the ENTIRE run to be accounted for: every + // migration either rolled_back or skipped_already_migrated (the latter + // stay on USDT by design). Anything else — including migrations still + // `complete` after a single-account rollback — blocks completion; a + // partial rollback must never flip the run-level config. + const unresolved = await migrationsRepo.listMigrationsByStatuses({ cutoverVersion, runId, - status: "rollback_started", + statuses: [ + "not_started", + "started", + "provisioned", + "balance_read", + "invoice_created", + "balance_move_sending", + "balance_move_sent", + "balance_move_verified", + "fee_reimbursement_invoice_created", + "fee_reimbursement_sending", + "fee_reimbursed", + "pointer_flipped", + "legacy_zero_verified", + "complete", + "failed", + "requires_operator_review", + "rollback_started", + ], + limit: 1, }) - if (inFlight instanceof Error) return inFlight - if (inFlight > 0) { + if (unresolved instanceof Error) return unresolved + if (unresolved.length > 0) { return new CashWalletCutoverInProgressError( - `${inFlight} migration(s) still in rollback_started`, - ) - } - - const review = await migrationsRepo.countByStatus({ - cutoverVersion, - runId, - status: "requires_operator_review", - }) - if (review instanceof Error) return review - if (review > 0) { - return new CashWalletMigrationFailedError( - `${review} migration(s) require operator review before completing rollback`, + `Migration ${unresolved[0].id} is still '${unresolved[0].status}'; run-level rollback completion requires every migration to be rolled_back or skipped_already_migrated. Single-account rollbacks do not complete the run.`, ) } diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts index c3cedfedf..219cfea93 100644 --- a/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/rollback-worker.spec.ts @@ -153,6 +153,22 @@ describe("rollback executor", () => { expect(svc.paymentService.payInvoice).not.toHaveBeenCalled() }) + it("never flips the pointer for pre-money migrations, even if the account defaults to USDT", async () => { + // No previousDefaultWalletId: the forward pipeline never flipped, so a + // USDT default came from elsewhere (e.g. native USDT-default signup). + const migration = { ...baseMigration } + const svc = services({ defaultWalletId: "destination-wallet-id" as WalletId }) + + const result = await executeCashWalletMigrationRollbackStep({ + migration, + migrationsRepo: statefulRepo(migration), + services: svc, + }) + + expect(svc.pointerService.flipDefaultWallet).not.toHaveBeenCalled() + expect(result).toMatchObject({ status: "rolled_back" }) + }) + it("restores the default pointer when the account still defaults to USDT", async () => { const migration = { ...baseMigration, diff --git a/test/flash/unit/app/cash-wallet-cutover/rollback.spec.ts b/test/flash/unit/app/cash-wallet-cutover/rollback.spec.ts new file mode 100644 index 000000000..47e07e02b --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/rollback.spec.ts @@ -0,0 +1,153 @@ +import { completePrimaryCashWalletRollback } from "@app/cash-wallet-cutover/rollback" +import { evaluateCashWalletCutoverGuard } from "@app/cash-wallet-cutover/guard" +import { + CashWalletCutoverInProgressError, + CashWalletMigrationFailedError, +} from "@app/cash-wallet-cutover/errors" + +const NOW = new Date("2026-07-04T00:00:00.000Z") + +const config = (state: CashWalletCutoverState): CashWalletCutoverConfig => ({ + state, + cutoverVersion: 3, + runId: "run-id", + updatedAt: NOW, +}) + +const migration = (status: CashWalletMigrationStatus) => + ({ + id: "migration-id", + accountId: "account-id" as AccountId, + legacyUsdWalletId: "legacy-wallet-id" as WalletId, + destinationUsdtWalletId: "destination-wallet-id" as WalletId, + cutoverVersion: 3, + runId: "run-id", + status, + idempotencyKey: "run-id:account-id", + attempts: 0, + updatedAt: NOW, + }) as CashWalletMigration + +const completionRepo = ({ + unresolved = [], +}: { + unresolved?: CashWalletMigration[] +} = {}) => + ({ + getConfig: jest.fn(async () => config("in_progress")), + listMigrationsByStatuses: jest.fn(async () => unresolved), + updateConfig: jest.fn(async (patch: Partial) => ({ + ...config("in_progress"), + ...patch, + })), + countByStatus: jest.fn(async () => 0), + }) as unknown as Parameters< + typeof completePrimaryCashWalletRollback + >[0]["migrationsRepo"] + +describe("completePrimaryCashWalletRollback", () => { + it("refuses when any migration is still complete (single-account rollback must not flip the run)", async () => { + const repo = completionRepo({ unresolved: [migration("complete")] }) + + const result = await completePrimaryCashWalletRollback({ + cutoverVersion: 3, + runId: "run-id", + actor: "operator-1", + now: NOW, + migrationsRepo: repo, + }) + + expect(result).toBeInstanceOf(CashWalletCutoverInProgressError) + expect((result as Error).message).toContain("'complete'") + }) + + it("refuses while any rollback is still in flight", async () => { + const repo = completionRepo({ unresolved: [migration("rollback_started")] }) + + const result = await completePrimaryCashWalletRollback({ + cutoverVersion: 3, + runId: "run-id", + actor: "operator-1", + now: NOW, + migrationsRepo: repo, + }) + + expect(result).toBeInstanceOf(CashWalletCutoverInProgressError) + }) + + it("completes when every migration is rolled_back or skipped_already_migrated", async () => { + const repo = completionRepo({ unresolved: [] }) + + const result = await completePrimaryCashWalletRollback({ + cutoverVersion: 3, + runId: "run-id", + actor: "operator-1", + now: NOW, + migrationsRepo: repo, + }) + + expect(result).toMatchObject({ state: "rolled_back", runId: "run-id" }) + }) +}) + +describe("cutover guard rollback routing", () => { + it("routes a rolled_back account to legacy USD even under a complete config (mode 3D)", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("complete"), + migration: migration("rolled_back"), + }), + ).toEqual({ route: "legacy_usd" }) + }) + + it("blocks an in-flight rollback under a complete config", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("complete"), + migration: migration("rollback_started"), + }), + ).toBeInstanceOf(CashWalletCutoverInProgressError) + }) + + it("still blanket-routes untouched accounts to USDT under a complete config", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("complete"), + migration: migration("complete"), + }), + ).toEqual({ route: "usdt" }) + }) + + it("routes rolled_back and absent migrations to legacy USD under a rolled_back config", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("rolled_back"), + migration: migration("rolled_back"), + }), + ).toEqual({ route: "legacy_usd" }) + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("rolled_back"), + migration: null, + }), + ).toEqual({ route: "legacy_usd" }) + }) + + it("keeps skipped_already_migrated accounts on USDT under a rolled_back config", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("rolled_back"), + migration: migration("skipped_already_migrated"), + }), + ).toEqual({ route: "usdt" }) + }) + + it("fails closed on statuses inconsistent with a rolled_back run", () => { + expect( + evaluateCashWalletCutoverGuard({ + cutover: config("rolled_back"), + migration: migration("complete"), + }), + ).toBeInstanceOf(CashWalletMigrationFailedError) + }) +})