diff --git a/apps/server/src/git/GitWorkflowService.test.ts b/apps/server/src/git/GitWorkflowService.test.ts index 2ea14b951..8455cf50e 100644 --- a/apps/server/src/git/GitWorkflowService.test.ts +++ b/apps/server/src/git/GitWorkflowService.test.ts @@ -2,12 +2,16 @@ import { assert, describe, expect, it, vi } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; -import { VcsRepositoryDetectionError } from "@t3tools/contracts"; +import { ProjectId, ThreadId, VcsRepositoryDetectionError } from "@t3tools/contracts"; import * as GitManager from "./GitManager.ts"; import * as GitWorkflowService from "./GitWorkflowService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; +import { + RollbackSagaRepository, + type RollbackSagaRecord, +} from "../persistence/Services/RollbackSagas.ts"; function makeLayer(input: { readonly detect: VcsDriverRegistry.VcsDriverRegistry["Service"]["detect"]; @@ -189,4 +193,77 @@ describe("GitWorkflowService", () => { ), ); }); + + it.effect( + "fails closed before a Git mutation when the workspace rollback lease is active", + () => { + const threadId = ThreadId.make("thread-git-fence"); + const projectId = ProjectId.make("project-git-fence"); + const record = { + operationId: "operation-git-fence", + requestEventId: "event-git-fence", + threadId, + projectId, + workspaceKey: "workspace-git-fence", + phase: "workspace-apply-started", + terminal: false, + ownerId: null, + version: 1, + state: { + operationId: "operation-git-fence", + requestEventId: "event-git-fence", + threadId, + projectId, + workspaceKey: "workspace-git-fence", + workspaceCwd: "/repo", + sourceRevision: 2, + targetRevision: 1, + sourceCheckpointRef: "refs/source" as never, + sourceCheckpointOid: "a".repeat(40), + targetCheckpointRef: "refs/target" as never, + targetCheckpointOid: "b".repeat(40), + targetCheckpointDigest: "target-tree", + providerInstanceId: "fake" as never, + sessionIncarnationId: "session" as never, + phase: "workspace-apply-started" as const, + attempt: 0, + lastErrorCode: null, + compensation: "none" as const, + cleanup: "pending" as const, + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: { leaf: "private" }, + desiredAnchorDigest: "target", + preimage: { path: "private" }, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", + }, + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", + } satisfies RollbackSagaRecord; + const repository = Layer.succeed(RollbackSagaRepository, { + listNonterminal: () => Effect.succeed([record]), + listNonterminalForFence: () => Effect.succeed([record]), + } as never); + const testLayer = makeLayer({ + detect: () => Effect.die("VCS detection must not run through a rollback fence"), + }).pipe(Layer.provideMerge(repository)); + + return Effect.gen(function* () { + const workflow = yield* GitWorkflowService.GitWorkflowService; + const result = yield* workflow.pullCurrentBranch("/repo").pipe(Effect.result); + assert.equal(result._tag, "Failure"); + if (result._tag === "Failure") { + expect(result.failure).toMatchObject({ + _tag: "GitCommandError", + command: "rollback-fence", + cwd: "/repo", + }); + } + }).pipe(Effect.provide(testLayer)); + }, + ); }); diff --git a/apps/server/src/git/GitWorkflowService.ts b/apps/server/src/git/GitWorkflowService.ts index a73aa59d5..0bab85777 100644 --- a/apps/server/src/git/GitWorkflowService.ts +++ b/apps/server/src/git/GitWorkflowService.ts @@ -1,6 +1,9 @@ +// @effect-diagnostics nodeBuiltinImport:off import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as NodeFS from "node:fs"; import { GitManagerError, @@ -31,6 +34,15 @@ import { import * as GitManager from "./GitManager.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts"; +import { RollbackSagaRepository } from "../persistence/Services/RollbackSagas.ts"; + +function canonicalWorkspacePath(cwd: string): string { + try { + return NodeFS.realpathSync(cwd); + } catch { + return cwd; + } +} export class GitWorkflowService extends Context.Service< GitWorkflowService, @@ -141,6 +153,44 @@ export const make = Effect.gen(function* () { const registry = yield* VcsDriverRegistry.VcsDriverRegistry; const git = yield* GitVcsDriver.GitVcsDriver; const gitManager = yield* GitManager.GitManager; + const rollbackRepository = yield* Effect.serviceOption(RollbackSagaRepository); + + const workspaceIsRollbackFenced = Effect.fn("GitWorkflowService.workspaceIsRollbackFenced")( + function* (cwd: string) { + if (Option.isNone(rollbackRepository)) return false; + const canonical = canonicalWorkspacePath(cwd); + const active = yield* rollbackRepository.value + .listNonterminalForFence() + .pipe(Effect.orElseSucceed(() => null)); + if (active === null) return true; + return active.some((record) => record.state.workspaceCwd === canonical); + }, + ); + const ensureMutationCommand = Effect.fn("GitWorkflowService.ensureMutationCommand")(function* ( + operation: string, + cwd: string, + ) { + if (yield* workspaceIsRollbackFenced(cwd)) { + return yield* new GitCommandError({ + operation, + command: "rollback-fence", + cwd, + detail: "The workspace is fenced by an active rollback operation.", + }); + } + }); + const ensureMutationWorkflow = Effect.fn("GitWorkflowService.ensureMutationWorkflow")(function* ( + operation: string, + cwd: string, + ) { + if (yield* workspaceIsRollbackFenced(cwd)) { + return yield* new GitManagerError({ + operation, + cwd, + detail: "The workspace is fenced by an active rollback operation.", + }); + } + }); const ensureGit = Effect.fn("GitWorkflowService.ensureGit")(function* ( operation: string, @@ -281,21 +331,24 @@ export const make = Effect.gen(function* () { invalidateRemoteStatus: gitManager.invalidateRemoteStatus, invalidateStatus: gitManager.invalidateStatus, pullCurrentBranch: (cwd) => - ensureGitCommand("GitWorkflowService.pullCurrentBranch", cwd).pipe( + ensureMutationCommand("GitWorkflowService.pullCurrentBranch", cwd).pipe( + Effect.andThen(ensureGitCommand("GitWorkflowService.pullCurrentBranch", cwd)), Effect.andThen(git.pullCurrentBranch(cwd)), ), runStackedAction: (input, options) => - ensureGit("GitWorkflowService.runStackedAction", input.cwd).pipe( + ensureMutationWorkflow("GitWorkflowService.runStackedAction", input.cwd).pipe( + Effect.andThen(ensureGit("GitWorkflowService.runStackedAction", input.cwd)), Effect.andThen(gitManager.runStackedAction(input, options)), ), resolvePullRequest: routeGitManager( "GitWorkflowService.resolvePullRequest", gitManager.resolvePullRequest, ), - preparePullRequestThread: routeGitManager( - "GitWorkflowService.preparePullRequestThread", - gitManager.preparePullRequestThread, - ), + preparePullRequestThread: (input) => + ensureMutationWorkflow("GitWorkflowService.preparePullRequestThread", input.cwd).pipe( + Effect.andThen(ensureGit("GitWorkflowService.preparePullRequestThread", input.cwd)), + Effect.andThen(gitManager.preparePullRequestThread(input)), + ), listRefs: (input) => detectGitRepositoryForCommand("GitWorkflowService.listRefs", input.cwd).pipe( Effect.flatMap((isGitRepository) => @@ -303,7 +356,8 @@ export const make = Effect.gen(function* () { ), ), createWorktree: (input) => - ensureGitCommand("GitWorkflowService.createWorktree", input.cwd).pipe( + ensureMutationCommand("GitWorkflowService.createWorktree", input.cwd).pipe( + Effect.andThen(ensureGitCommand("GitWorkflowService.createWorktree", input.cwd)), Effect.andThen(git.createWorktree(input)), ), fetchRemote: (input) => @@ -319,7 +373,8 @@ export const make = Effect.gen(function* () { Effect.andThen(git.resolveRemoteTrackingCommit(input)), ), removeWorktree: (input) => - ensureGitCommand("GitWorkflowService.removeWorktree", input.cwd).pipe( + ensureMutationCommand("GitWorkflowService.removeWorktree", input.cwd).pipe( + Effect.andThen(ensureGitCommand("GitWorkflowService.removeWorktree", input.cwd)), Effect.andThen(git.removeWorktree(input)), ), pruneWorktrees: (input) => @@ -327,15 +382,18 @@ export const make = Effect.gen(function* () { Effect.andThen(git.pruneWorktrees(input)), ), createRef: (input) => - ensureGitCommand("GitWorkflowService.createRef", input.cwd).pipe( + ensureMutationCommand("GitWorkflowService.createRef", input.cwd).pipe( + Effect.andThen(ensureGitCommand("GitWorkflowService.createRef", input.cwd)), Effect.andThen(git.createRef(input)), ), switchRef: (input) => - ensureGitCommand("GitWorkflowService.switchRef", input.cwd).pipe( + ensureMutationCommand("GitWorkflowService.switchRef", input.cwd).pipe( + Effect.andThen(ensureGitCommand("GitWorkflowService.switchRef", input.cwd)), Effect.andThen(Effect.scoped(git.switchRef(input))), ), renameBranch: (input) => - ensureGit("GitWorkflowService.renameBranch", input.cwd).pipe( + ensureMutationWorkflow("GitWorkflowService.renameBranch", input.cwd).pipe( + Effect.andThen(ensureGit("GitWorkflowService.renameBranch", input.cwd)), Effect.andThen(git.renameBranch(input)), ), }); diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.ts index 6a1916f51..e0804e6d5 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts @@ -5,6 +5,7 @@ import { MessageId, type ProjectId, type ProviderInstanceId, + type RuntimeSessionId, ThreadId, TurnId, type OrchestrationEvent, @@ -17,7 +18,6 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import type * as PlatformError from "effect/PlatformError"; import * as Stream from "effect/Stream"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { isTemporaryWorktreeBranch } from "@t3tools/shared/git"; @@ -35,11 +35,12 @@ import { forkParked } from "../../serverActivation.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; -import type { CheckpointStoreError } from "../../checkpointing/Errors.ts"; -import type { OrchestrationDispatchError } from "../Errors.ts"; import { isGitRepository } from "../../git/Utils.ts"; import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; +import { RollbackSagaRepository } from "../../persistence/Services/RollbackSagas.ts"; +import { RollbackSagaRunner } from "../../rollback/RollbackSagaRunner.ts"; +import { RollbackWorkspace } from "../../rollback/RollbackWorkspace.ts"; const nowIso = Effect.map(DateTime.now, DateTime.formatIso); @@ -51,6 +52,11 @@ type ReactorInput = | { readonly source: "domain"; readonly event: OrchestrationEvent; + } + | { + readonly source: "saga"; + readonly operationId: string; + readonly recovering: boolean; }; function toTurnId(value: string | undefined): TurnId | null { @@ -77,7 +83,7 @@ function checkpointStatusFromRuntime(status: string | undefined): "ready" | "mis } } -const make = Effect.gen(function* () { +export const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const randomUUID = crypto.randomUUIDv4; const serverEventId = randomUUID.pipe(Effect.map(EventId.make)); @@ -93,6 +99,9 @@ const make = Effect.gen(function* () { const receiptBus = yield* RuntimeReceiptBus; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; const vcsStatusBroadcaster = yield* VcsStatusBroadcaster; + const rollbackRepository = yield* Effect.serviceOption(RollbackSagaRepository); + const rollbackRunner = yield* Effect.serviceOption(RollbackSagaRunner); + const rollbackWorkspace = yield* Effect.serviceOption(RollbackWorkspace); const appendRevertFailureActivity = (input: { readonly threadId: ThreadId; @@ -164,6 +173,7 @@ const make = Effect.gen(function* () { readonly threadId: ThreadId; readonly cwd: string; readonly providerInstanceId: ProviderInstanceId; + readonly sessionIncarnationId: RuntimeSessionId | undefined; }> > { const sessions = yield* providerService.listSessions(); @@ -174,6 +184,7 @@ const make = Effect.gen(function* () { cwd: session.cwd, providerInstanceId: session.providerInstanceId ?? defaultInstanceIdForDriver(session.provider), + sessionIncarnationId: session.sessionIncarnationId, }) : Option.none(); }); @@ -231,6 +242,52 @@ const make = Effect.gen(function* () { return cwd; }); + const capturePrivateCheckpointAnchor = Effect.fn("capturePrivateCheckpointAnchor")( + function* (input: { + readonly threadId: ThreadId; + readonly cwd: string; + readonly checkpointTurnCount: number; + readonly checkpointRef: ReturnType; + readonly capturedAt: string; + }) { + const session = yield* resolveSessionRuntimeForThread(input.threadId); + if ( + Option.isNone(session) || + session.value.sessionIncarnationId === undefined || + Option.isNone(rollbackRepository) || + Option.isNone(rollbackWorkspace) + ) + return; + const capabilities = yield* providerService + .getCapabilities(session.value.providerInstanceId) + .pipe(Effect.option); + if (Option.isNone(capabilities) || capabilities.value.conversationRollback !== "absolute") + return; + if ( + providerService.hasAbsoluteConversationRollback === undefined || + providerService.captureConversationAnchor === undefined || + !(yield* providerService.hasAbsoluteConversationRollback(input.threadId)) + ) + return; + const checkpoint = yield* rollbackWorkspace.value.resolveCheckpoint({ + cwd: input.cwd, + checkpointRef: input.checkpointRef, + }); + const anchor = yield* providerService.captureConversationAnchor(input.threadId); + yield* rollbackRepository.value.putCheckpointAnchor({ + threadId: input.threadId, + checkpointTurnCount: input.checkpointTurnCount, + providerInstanceId: session.value.providerInstanceId, + sessionIncarnationId: session.value.sessionIncarnationId, + checkpointRef: input.checkpointRef, + checkpointOid: checkpoint.oid, + anchor: anchor.anchor, + anchorDigest: anchor.digest, + capturedAt: input.capturedAt, + }); + }, + ); + // Shared tail for both capture paths: creates the git checkpoint ref, diffs // it against the previous turn, then dispatches the domain events to update // the orchestration read model. @@ -270,6 +327,13 @@ const make = Effect.gen(function* () { cwd: input.cwd, checkpointRef: targetCheckpointRef, }); + yield* capturePrivateCheckpointAnchor({ + threadId: input.threadId, + cwd: input.cwd, + checkpointTurnCount: input.turnCount, + checkpointRef: targetCheckpointRef, + capturedAt: input.createdAt, + }); // Refresh the workspace entry index so the @-mention file picker // reflects files created or deleted during this turn. @@ -707,6 +771,14 @@ const make = Effect.gen(function* () { ) { const now = DateTime.formatIso(yield* DateTime.now); + const admitted = Option.isSome(rollbackRepository) + ? yield* rollbackRepository.value.getByRequestEvent(event.eventId).pipe(Effect.option) + : Option.none(); + if (Option.isSome(admitted) && Option.isSome(admitted.value) && Option.isSome(rollbackRunner)) { + yield* rollbackRunner.value.run(admitted.value.value.operationId, false); + return; + } + const thread = yield* resolveThreadDetail(event.payload.threadId); if (!thread) { yield* appendRevertFailureActivity({ @@ -846,14 +918,14 @@ const make = Effect.gen(function* () { } }); - const processInput = ( - input: ReactorInput, - ): Effect.Effect< - void, - CheckpointStoreError | OrchestrationDispatchError | PlatformError.PlatformError, - never - > => - input.source === "domain" ? processDomainEvent(input.event) : processRuntimeEvent(input.event); + const processInput = (input: ReactorInput) => + input.source === "domain" + ? processDomainEvent(input.event) + : input.source === "runtime" + ? processRuntimeEvent(input.event) + : Option.isSome(rollbackRunner) + ? rollbackRunner.value.run(input.operationId, input.recovering) + : Effect.void; const processInputSafely = (input: ReactorInput) => processInput(input).pipe( @@ -863,8 +935,8 @@ const make = Effect.gen(function* () { } return Effect.logWarning("checkpoint reactor failed to process input", { source: input.source, - eventType: input.event.type, - cause: Cause.pretty(cause), + eventType: input.source === "saga" ? "rollback.saga.reconcile" : input.event.type, + cause: input.source === "saga" ? "rollback saga step failed" : Cause.pretty(cause), }); }), ); @@ -872,6 +944,17 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processInputSafely); const start: CheckpointReactorShape["start"] = Effect.fn("start")(function* () { + if (Option.isSome(rollbackRepository) && Option.isSome(rollbackRunner)) { + yield* rollbackRepository.value.clearOwnersForStartup().pipe(Effect.orDie); + const pendingRollbacks = yield* rollbackRepository.value.listNonterminal().pipe(Effect.orDie); + yield* Effect.forEach( + pendingRollbacks, + (record) => + worker.enqueue({ source: "saga", operationId: record.operationId, recovering: true }), + { concurrency: 1, discard: true }, + ); + } + yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if ( diff --git a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts index 17f15deda..feb957bce 100644 --- a/apps/server/src/orchestration/Layers/OrchestrationEngine.ts +++ b/apps/server/src/orchestration/Layers/OrchestrationEngine.ts @@ -1,3 +1,4 @@ +// @effect-diagnostics nodeBuiltinImport:off import type { OrchestrationClientOrigin, OrchestrationEvent, @@ -5,7 +6,7 @@ import type { ProjectId, ThreadId, } from "@t3tools/contracts"; -import { OrchestrationCommand } from "@t3tools/contracts"; +import { CommandId, OrchestrationCommand } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Crypto from "effect/Crypto"; @@ -21,6 +22,7 @@ import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import * as NodeFS from "node:fs"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import { @@ -41,6 +43,8 @@ import { } from "../Errors.ts"; import { decideOrchestrationCommand } from "../decider.ts"; import { createEmptyReadModel, projectEvent } from "../projector.ts"; +import { RollbackAdmission } from "../../rollback/RollbackAdmission.ts"; +import { RollbackSagaRepository } from "../../persistence/Services/RollbackSagas.ts"; import { OrchestrationProjectionPipeline } from "../Services/ProjectionPipeline.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { @@ -53,6 +57,14 @@ const isOrchestrationCommandPreviouslyRejectedError = Schema.is( const isOrchestrationCommandIdConflictError = Schema.is(OrchestrationCommandIdConflictError); const isOrchestrationCommandInvariantError = Schema.is(OrchestrationCommandInvariantError); +function canonicalWorkspacePath(cwd: string): string { + try { + return NodeFS.realpathSync(cwd); + } catch { + return cwd; + } +} + interface CommandEnvelope { command: OrchestrationCommand; origin: OrchestrationClientOrigin | undefined; @@ -87,6 +99,8 @@ const makeOrchestrationEngine = Effect.gen(function* () { const projectionPipeline = yield* OrchestrationProjectionPipeline; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; const crypto = yield* Crypto.Crypto; + const rollbackAdmission = yield* Effect.serviceOption(RollbackAdmission); + const rollbackRepository = yield* Effect.serviceOption(RollbackSagaRepository); const nowIso = Effect.map(DateTime.now, DateTime.formatIso); let commandReadModel = createEmptyReadModel(yield* nowIso); @@ -106,6 +120,74 @@ const makeOrchestrationEngine = Effect.gen(function* () { return nextReadModel; }); + const assertRollbackFenceAllows = Effect.fn("OrchestrationEngine.assertRollbackFenceAllows")( + function* (command: OrchestrationCommand) { + if (Option.isNone(rollbackRepository)) return; + const active = yield* rollbackRepository.value.listNonterminalForFence().pipe( + Effect.mapError( + () => + new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: "Rollback mutation fence could not be verified.", + }), + ), + ); + if (active.length === 0) return; + if ( + command.type === "project.create" || + command.type === "project.meta.update" || + command.type === "project.delete" + ) { + if ( + command.type !== "project.create" && + active.some((record) => record.projectId === command.projectId) + ) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: "This project is fenced by an active rollback operation.", + }); + } + return; + } + const safeForOwnedThread = + command.type === "thread.rollback.status.set" || + command.type === "thread.revert.complete" || + (command.type === "thread.session.set" && + ["idle", "ready", "interrupted", "stopped", "error"].includes(command.session.status)); + const owned = active.find((record) => record.threadId === command.threadId); + if (owned !== undefined && !safeForOwnedThread) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: "This thread is fenced by an active rollback operation.", + }); + } + if ( + command.type !== "thread.turn.start" && + command.type !== "thread.input-queue.follow-up" && + command.type !== "thread.approval.respond" && + command.type !== "thread.user-input.respond" && + command.type !== "thread.turn.diff.complete" && + command.type !== "thread.checkpoint.revert" + ) + return; + const thread = commandReadModel.threads.find( + (candidate) => candidate.id === command.threadId, + ); + const project = + thread === undefined + ? undefined + : commandReadModel.projects.find((candidate) => candidate.id === thread.projectId); + if (!thread || !project) return; + const candidateCwd = canonicalWorkspacePath(thread.worktreePath ?? project.workspaceRoot); + if (active.some((record) => record.state.workspaceCwd === candidateCwd)) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: "This workspace is fenced by another thread's active rollback operation.", + }); + } + }, + ); + const processEnvelope = (envelope: CommandEnvelope): Effect.Effect => { const dispatchStartSequence = commandReadModel.snapshotSequence; let processingStartedAtMs = 0; @@ -169,6 +251,8 @@ const makeOrchestrationEngine = Effect.gen(function* () { }); } + yield* assertRollbackFenceAllows(envelope.command); + const eventBase = yield* decideOrchestrationCommand({ command: envelope.command, readModel: commandReadModel, @@ -184,7 +268,50 @@ const makeOrchestrationEngine = Effect.gen(function* () { }), ), ); - const plannedEvents = Array.isArray(eventBase) ? eventBase : [eventBase]; + const commandEvents = Array.isArray(eventBase) ? eventBase : [eventBase]; + const preparedRollback = + envelope.command.type === "thread.checkpoint.revert" && + Option.isSome(rollbackAdmission) && + commandEvents[0]?.type === "thread.checkpoint-revert-requested" + ? yield* rollbackAdmission.value.prepare({ + command: envelope.command, + readModel: commandReadModel, + requestEventId: commandEvents[0].eventId, + }) + : Option.none(); + const pendingRollbackEvent = Option.isSome(preparedRollback) + ? yield* decideOrchestrationCommand({ + command: { + type: "thread.rollback.status.set", + commandId: CommandId.make( + `server:rollback-admitted:${preparedRollback.value.operationId}`, + ), + threadId: preparedRollback.value.threadId, + status: "pending", + createdAt: preparedRollback.value.createdAt, + }, + readModel: commandReadModel, + }).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.mapError( + (cause) => + new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: "Failed to persist the durable rollback status.", + cause, + }), + ), + ) + : null; + const plannedEvents = + pendingRollbackEvent === null + ? commandEvents + : [ + ...commandEvents, + ...(Array.isArray(pendingRollbackEvent) + ? pendingRollbackEvent + : [pendingRollbackEvent]), + ]; // Stamp the dispatching client's origin onto every event the command // produced. The decider stays pure; attribution is an engine concern. const eventBases = @@ -197,6 +324,24 @@ const makeOrchestrationEngine = Effect.gen(function* () { const committedCommand = yield* sql .withTransaction( Effect.gen(function* () { + if (Option.isSome(preparedRollback)) { + if (Option.isNone(rollbackRepository)) { + return yield* new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: "Durable rollback persistence is unavailable.", + }); + } + yield* rollbackRepository.value.admit(preparedRollback.value).pipe( + Effect.mapError( + () => + new OrchestrationCommandInvariantError({ + commandType: envelope.command.type, + detail: + "Rollback admission lost its operation or workspace lease compare-and-set.", + }), + ), + ); + } const committedEvents: OrchestrationEvent[] = []; let nextCommandReadModel = commandReadModel; diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 465771332..12b5f8b63 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -644,6 +644,8 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti worktreePath: event.payload.worktreePath, linkedPullRequest: null, latestTurnId: null, + rollbackStatus: null, + rollbackUpdatedAt: null, createdAt: event.payload.createdAt, updatedAt: event.payload.updatedAt, archivedAt: null, @@ -961,6 +963,21 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti return; } + case "thread.rollback-status-updated": { + const existingRow = yield* projectionThreadRepository.getById({ + threadId: event.payload.threadId, + }); + if (Option.isNone(existingRow)) return; + yield* projectionThreadRepository.upsert({ + ...existingRow.value, + rollbackStatus: event.payload.status, + rollbackUpdatedAt: event.payload.updatedAt, + updatedAt: event.payload.updatedAt, + }); + yield* refreshThreadShellSummary(event.payload.threadId); + return; + } + case "thread.reverted": { const existingRow = yield* projectionThreadRepository.getById({ threadId: event.payload.threadId, @@ -993,6 +1010,8 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti yield* projectionThreadRepository.upsert({ ...existingRow.value, latestTurnId, + rollbackStatus: null, + rollbackUpdatedAt: event.occurredAt, updatedAt: event.occurredAt, }); yield* refreshThreadShellSummary(event.payload.threadId); diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts index 89c1e723f..b6a03b487 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.test.ts @@ -328,6 +328,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { }, interactionMode: "default", runtimeMode: "full-access", + rollbackStatus: null, branch: null, worktreePath: null, linkedPullRequest: { @@ -458,6 +459,7 @@ projectionSnapshotLayer("ProjectionSnapshotQuery", (it) => { }, interactionMode: "default", runtimeMode: "full-access", + rollbackStatus: null, branch: null, worktreePath: null, linkedPullRequest: { diff --git a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts index b44436ba8..2fcae92dd 100644 --- a/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts +++ b/apps/server/src/orchestration/Layers/ProjectionSnapshotQuery.ts @@ -507,6 +507,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", @@ -546,6 +548,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", @@ -587,6 +591,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", @@ -1107,6 +1113,8 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", @@ -1416,6 +1424,7 @@ const makeProjectionSnapshotQuery = Effect.gen(function* () { 'thread.proposed-plan-upserted', 'thread.activity-appended', 'thread.turn-diff-completed', + 'thread.rollback-status-updated', 'thread.reverted', 'thread.session-set' ) @@ -2080,6 +2089,10 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: row.linkedPullRequest }), latestTurn: latestTurnByThread.get(row.threadId) ?? null, + rollbackStatus: + row.rollbackStatus == null || row.rollbackUpdatedAt == null + ? null + : { state: row.rollbackStatus, updatedAt: row.rollbackUpdatedAt }, createdAt: row.createdAt, updatedAt: row.updatedAt, archivedAt: row.archivedAt, @@ -2292,6 +2305,10 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: row.linkedPullRequest }), latestTurn: latestTurnByThread.get(row.threadId) ?? null, + rollbackStatus: + row.rollbackStatus == null || row.rollbackUpdatedAt == null + ? null + : { state: row.rollbackStatus, updatedAt: row.rollbackUpdatedAt }, createdAt: row.createdAt, updatedAt: row.updatedAt, archivedAt: row.archivedAt, @@ -2433,6 +2450,10 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: row.linkedPullRequest }), latestTurn: latestTurnByThread.get(row.threadId) ?? null, + rollbackStatus: + row.rollbackStatus == null || row.rollbackUpdatedAt == null + ? null + : { state: row.rollbackStatus, updatedAt: row.rollbackUpdatedAt }, createdAt: row.createdAt, updatedAt: row.updatedAt, archivedAt: row.archivedAt, @@ -2583,6 +2604,10 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: row.linkedPullRequest }), latestTurn: latestTurnByThread.get(row.threadId) ?? null, + rollbackStatus: + row.rollbackStatus == null || row.rollbackUpdatedAt == null + ? null + : { state: row.rollbackStatus, updatedAt: row.rollbackUpdatedAt }, createdAt: row.createdAt, updatedAt: row.updatedAt, archivedAt: row.archivedAt, @@ -2878,6 +2903,13 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: threadRow.value.linkedPullRequest }), latestTurn: Option.isSome(latestTurnRow) ? mapLatestTurn(latestTurnRow.value) : null, + rollbackStatus: + threadRow.value.rollbackStatus == null || threadRow.value.rollbackUpdatedAt == null + ? null + : { + state: threadRow.value.rollbackStatus, + updatedAt: threadRow.value.rollbackUpdatedAt, + }, createdAt: threadRow.value.createdAt, updatedAt: threadRow.value.updatedAt, archivedAt: threadRow.value.archivedAt, @@ -3117,6 +3149,13 @@ pending_approval_requests AS ( ? {} : { linkedPullRequest: threadRow.value.linkedPullRequest }), latestTurn: Option.isSome(latestTurnRow) ? mapLatestTurn(latestTurnRow.value) : null, + rollbackStatus: + threadRow.value.rollbackStatus == null || threadRow.value.rollbackUpdatedAt == null + ? null + : { + state: threadRow.value.rollbackStatus, + updatedAt: threadRow.value.rollbackUpdatedAt, + }, createdAt: threadRow.value.createdAt, updatedAt: threadRow.value.updatedAt, archivedAt: threadRow.value.archivedAt, diff --git a/apps/server/src/orchestration/Layers/RollbackAdmissionAtomic.test.ts b/apps/server/src/orchestration/Layers/RollbackAdmissionAtomic.test.ts new file mode 100644 index 000000000..bb4f3f7a5 --- /dev/null +++ b/apps/server/src/orchestration/Layers/RollbackAdmissionAtomic.test.ts @@ -0,0 +1,235 @@ +// @effect-diagnostics preferSchemaOverJson:off +import { + CommandId, + DEFAULT_PROVIDER_INTERACTION_MODE, + MessageId, + ProjectId, + ProviderInstanceId, + RuntimeSessionId, + ThreadId, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +import { checkpointRefForThreadTurn } from "../../checkpointing/Utils.ts"; +import { ServerConfig } from "../../config.ts"; +import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; +import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts"; +import { RollbackSagaRepositoryLive } from "../../persistence/Layers/RollbackSagas.ts"; +import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; +import { RollbackSagaRepository } from "../../persistence/Services/RollbackSagas.ts"; +import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts"; +import { RollbackAdmission } from "../../rollback/RollbackAdmission.ts"; +import * as ThreadBackgroundLiveness from "../ThreadBackgroundLiveness.ts"; +import * as ThreadPlanProgress from "../ThreadPlanProgress.ts"; +import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; +import { OrchestrationEngineLive } from "./OrchestrationEngine.ts"; +import { OrchestrationProjectionPipelineLive } from "./ProjectionPipeline.ts"; +import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQuery.ts"; + +const now = "2026-08-31T00:00:00.000Z"; +const threadId = ThreadId.make("thread-atomic-rollback"); +const siblingThreadId = ThreadId.make("thread-atomic-sibling"); +const projectId = ProjectId.make("project-atomic-rollback"); +const providerInstanceId = ProviderInstanceId.make("fake-absolute"); +const sessionIncarnationId = RuntimeSessionId.make("session-atomic-rollback"); +const operationId = "operation-atomic-rollback"; +const privateCanary = "PRIVATE_ATOMIC_TARGET_CANARY"; + +const admission = Layer.succeed(RollbackAdmission, { + prepare: ({ requestEventId }) => + Effect.succeed( + Option.some({ + operationId, + requestEventId, + threadId, + projectId, + workspaceKey: "workspace-atomic", + workspaceCwd: "/workspace/atomic", + sourceRevision: 2, + targetRevision: 1, + sourceCheckpointRef: checkpointRefForThreadTurn(threadId, 2), + sourceCheckpointOid: "2".repeat(40), + targetCheckpointRef: checkpointRefForThreadTurn(threadId, 1), + targetCheckpointOid: "1".repeat(40), + targetCheckpointDigest: "target-tree", + providerInstanceId, + sessionIncarnationId, + phase: "source-anchor-capture-started" as const, + attempt: 0, + lastErrorCode: null, + compensation: "none" as const, + cleanup: "pending" as const, + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: { leafId: privateCanary }, + desiredAnchorDigest: "target-anchor", + preimage: null, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: now, + updatedAt: now, + }), + ), +}); + +const engine = OrchestrationEngineLive.pipe( + Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationProjectionPipelineLive), + Layer.provideMerge(admission), + Layer.provideMerge(RollbackSagaRepositoryLive), +); +const app = Layer.mergeAll( + engine, + OrchestrationProjectionSnapshotQueryLive, + RollbackSagaRepositoryLive, +).pipe( + Layer.provide(ThreadBackgroundLiveness.layer), + Layer.provide(ThreadPlanProgress.layer), + Layer.provide(OrchestrationEventStoreLive), + Layer.provide(OrchestrationCommandReceiptRepositoryLive), + Layer.provide(RepositoryIdentityResolver.layer), + Layer.provide(SqlitePersistenceMemory), + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { + prefix: "t3-rollback-atomic-test-", + }), + ), + Layer.provideMerge(NodeServices.layer), +); + +const layer = it.layer(app); +layer("durable rollback admission", (it) => { + it.effect( + "atomically admits private state, publishes pending, and fences concurrent mutations", + () => + Effect.gen(function* () { + const orchestration = yield* OrchestrationEngineService; + const snapshots = yield* ProjectionSnapshotQuery; + const repository = yield* RollbackSagaRepository; + yield* orchestration.dispatch({ + type: "project.create", + commandId: CommandId.make("command-atomic-project"), + projectId, + title: "Atomic rollback", + workspaceRoot: "/workspace/atomic", + defaultModelSelection: { instanceId: providerInstanceId, model: "fake" }, + createdAt: now, + }); + for (const [id, title] of [ + [threadId, "Atomic rollback"], + [siblingThreadId, "Atomic rollback sibling"], + ] as const) { + yield* orchestration.dispatch({ + type: "thread.create", + commandId: CommandId.make(`command-create-${id}`), + threadId: id, + projectId, + title, + modelSelection: { instanceId: providerInstanceId, model: "fake" }, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: now, + }); + } + + yield* orchestration.dispatch({ + type: "thread.checkpoint.revert", + commandId: CommandId.make("command-atomic-revert"), + threadId, + turnCount: 1, + expectedSourceRevision: 2, + createdAt: now, + }); + + const fencedCommands = [ + orchestration.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("command-fenced-send"), + threadId, + message: { + messageId: MessageId.make("message-fenced-send"), + role: "user", + text: "must not start", + attachments: [], + }, + runtimeMode: "full-access", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + createdAt: now, + }), + orchestration.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("command-fenced-sibling-send"), + threadId: siblingThreadId, + message: { + messageId: MessageId.make("message-fenced-sibling-send"), + role: "user", + text: "must not mutate shared workspace", + attachments: [], + }, + runtimeMode: "full-access", + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + createdAt: now, + }), + orchestration.dispatch({ + type: "thread.session.stop", + commandId: CommandId.make("command-fenced-stop"), + threadId, + createdAt: now, + }), + orchestration.dispatch({ + type: "thread.checkpoint.revert", + commandId: CommandId.make("command-fenced-second-revert"), + threadId, + turnCount: 0, + expectedSourceRevision: 2, + createdAt: now, + }), + orchestration.dispatch({ + type: "project.meta.update", + commandId: CommandId.make("command-fenced-project"), + projectId, + title: "must not change", + }), + ]; + for (const [index, command] of fencedCommands.entries()) { + const result = yield* command.pipe(Effect.result); + assert.equal(result._tag, "Failure", `fenced command index ${index}`); + } + + assert.isTrue(Option.isSome(yield* repository.get(operationId))); + const snapshot = yield* snapshots.getSnapshot(); + const thread = snapshot.threads.find((candidate) => candidate.id === threadId); + assert.deepEqual(thread?.rollbackStatus, { state: "pending", updatedAt: now }); + const shell = yield* snapshots.getShellSnapshot(); + assert.deepEqual( + shell.threads.find((candidate) => candidate.id === threadId)?.rollbackStatus, + { state: "pending", updatedAt: now }, + ); + + const events = yield* Stream.runCollect(orchestration.readEvents(0)).pipe( + Effect.map((chunk) => Array.from(chunk)), + ); + assert.deepEqual( + events.slice(0, 5).map((event) => event.type), + [ + "project.created", + "thread.created", + "thread.created", + "thread.checkpoint-revert-requested", + "thread.rollback-status-updated", + ], + ); + assert.notInclude(JSON.stringify(events), privateCanary); + assert.notInclude(JSON.stringify(snapshot), privateCanary); + }), + ); +}); diff --git a/apps/server/src/orchestration/Layers/RollbackReconciliation.test.ts b/apps/server/src/orchestration/Layers/RollbackReconciliation.test.ts new file mode 100644 index 000000000..571d89f33 --- /dev/null +++ b/apps/server/src/orchestration/Layers/RollbackReconciliation.test.ts @@ -0,0 +1,117 @@ +import { ProjectId, ThreadId } from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Stream from "effect/Stream"; + +import { CheckpointStore } from "../../checkpointing/CheckpointStore.ts"; +import { + RollbackSagaRepository, + type RollbackSagaRecord, +} from "../../persistence/Services/RollbackSagas.ts"; +import { ProviderService } from "../../provider/Services/ProviderService.ts"; +import { RollbackSagaRunner } from "../../rollback/RollbackSagaRunner.ts"; +import { RollbackWorkspace } from "../../rollback/RollbackWorkspace.ts"; +import { VcsStatusBroadcaster } from "../../vcs/VcsStatusBroadcaster.ts"; +import { WorkspaceEntries } from "../../workspace/WorkspaceEntries.ts"; +import { make as makeCheckpointReactor } from "./CheckpointReactor.ts"; +import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; +import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; +import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; + +const operationId = "startup-operation"; +const threadId = ThreadId.make("startup-thread"); +const projectId = ProjectId.make("startup-project"); +const pending = { + operationId, + requestEventId: "startup-event", + threadId, + projectId, + workspaceKey: "startup-workspace", + phase: "workspace-apply-started", + terminal: false, + ownerId: "dead-process-owner", + version: 4, + state: { + operationId, + requestEventId: "startup-event", + threadId, + projectId, + workspaceKey: "startup-workspace", + workspaceCwd: "/startup/workspace", + sourceRevision: 2, + targetRevision: 1, + sourceCheckpointRef: "refs/t3/checkpoints/source" as never, + sourceCheckpointOid: "a".repeat(40), + targetCheckpointRef: "refs/t3/checkpoints/target" as never, + targetCheckpointOid: "b".repeat(40), + targetCheckpointDigest: "tree-target", + providerInstanceId: "fake" as never, + sessionIncarnationId: "session" as never, + phase: "workspace-apply-started" as const, + attempt: 0, + lastErrorCode: null, + compensation: "none" as const, + cleanup: "pending" as const, + sourceAnchor: { leaf: "private-source" }, + sourceAnchorDigest: "source", + desiredAnchor: { leaf: "private-target" }, + desiredAnchorDigest: "target", + preimage: { path: "private" }, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", + }, + createdAt: "2026-08-31T00:00:00.000Z", + updatedAt: "2026-08-31T00:00:00.000Z", +} satisfies RollbackSagaRecord; + +it.effect("clears stale owners and enqueues every nonterminal rollback during startup", () => + Effect.gen(function* () { + const calls: string[] = []; + const repository = { + clearOwnersForStartup: () => + Effect.sync(() => { + calls.push("clear-owners"); + }), + listNonterminal: () => + Effect.sync(() => { + calls.push("list-nonterminal"); + return [pending]; + }), + }; + const runner = { + run: (id: string, recovering: boolean) => + Effect.sync(() => { + calls.push(`run:${id}:${recovering}`); + }), + }; + const reactor = yield* makeCheckpointReactor.pipe( + Effect.provideService(OrchestrationEngineService, { + dispatch: () => Effect.succeed({ sequence: 1 }), + readEvents: () => Stream.empty, + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(1), + } as never), + Effect.provideService(ProjectionSnapshotQuery, {} as never), + Effect.provideService(ProviderService, { streamEvents: Stream.empty } as never), + Effect.provideService(CheckpointStore, {} as never), + Effect.provideService(RuntimeReceiptBus, { + publish: () => Effect.void, + streamEventsForTest: Stream.empty, + }), + Effect.provideService(WorkspaceEntries, {} as never), + Effect.provideService(VcsStatusBroadcaster, {} as never), + Effect.provideService(RollbackSagaRepository, repository as never), + Effect.provideService(RollbackSagaRunner, runner), + Effect.provideService(RollbackWorkspace, {} as never), + Effect.provide(NodeServices.layer), + ); + + yield* reactor.start(); + yield* reactor.drain; + assert.deepEqual(calls, ["clear-owners", "list-nonterminal", `run:${operationId}:true`]); + }), +); diff --git a/apps/server/src/orchestration/Services/RuntimeReceiptBus.ts b/apps/server/src/orchestration/Services/RuntimeReceiptBus.ts index 0b880ee69..da6bb02f8 100644 --- a/apps/server/src/orchestration/Services/RuntimeReceiptBus.ts +++ b/apps/server/src/orchestration/Services/RuntimeReceiptBus.ts @@ -49,7 +49,16 @@ export const TurnProcessingQuiescedReceipt = Schema.Struct({ }); export type TurnProcessingQuiescedReceipt = typeof TurnProcessingQuiescedReceipt.Type; +export const RollbackSagaPhaseReceipt = Schema.Struct({ + type: Schema.Literal("rollback.saga.phase"), + operationId: Schema.String, + phase: Schema.String, + createdAt: IsoDateTime, +}); +export type RollbackSagaPhaseReceipt = typeof RollbackSagaPhaseReceipt.Type; + export const OrchestrationRuntimeReceipt = Schema.Union([ + RollbackSagaPhaseReceipt, CheckpointBaselineCapturedReceipt, CheckpointDiffFinalizedReceipt, TurnProcessingQuiescedReceipt, diff --git a/apps/server/src/orchestration/decider.ts b/apps/server/src/orchestration/decider.ts index 1a2e0c1fa..244c99214 100644 --- a/apps/server/src/orchestration/decider.ts +++ b/apps/server/src/orchestration/decider.ts @@ -1884,11 +1884,25 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" } case "thread.revert.complete": { - yield* requireThread({ + const thread = yield* requireThread({ readModel, command, threadId: command.threadId, }); + const currentRevision = thread.checkpoints.reduce( + (maximum, checkpoint) => Math.max(maximum, checkpoint.checkpointTurnCount), + 0, + ); + if ( + currentRevision !== command.sourceRevision || + command.targetRevision !== command.turnCount || + command.targetRevision >= command.sourceRevision + ) { + return yield* new OrchestrationCommandInvariantError({ + commandType: command.type, + detail: "Rollback completion lost its exact source/target revision compare-and-set.", + }); + } return { ...(yield* withEventBase({ aggregateKind: "thread", @@ -1900,6 +1914,27 @@ export const decideOrchestrationCommand = Effect.fn("decideOrchestrationCommand" payload: { threadId: command.threadId, turnCount: command.turnCount, + operationId: command.operationId, + sourceRevision: command.sourceRevision, + targetRevision: command.targetRevision, + }, + }; + } + + case "thread.rollback.status.set": { + yield* requireThread({ readModel, command, threadId: command.threadId }); + return { + ...(yield* withEventBase({ + aggregateKind: "thread", + aggregateId: command.threadId, + occurredAt: command.createdAt, + commandId: command.commandId, + })), + type: "thread.rollback-status-updated", + payload: { + threadId: command.threadId, + status: command.status, + updatedAt: command.createdAt, }, }; } diff --git a/apps/server/src/orchestration/projector.ts b/apps/server/src/orchestration/projector.ts index c5bfd9591..a8609282f 100644 --- a/apps/server/src/orchestration/projector.ts +++ b/apps/server/src/orchestration/projector.ts @@ -735,6 +735,18 @@ export function projectEvent( }; }); + case "thread.rollback-status-updated": + return Effect.succeed({ + ...nextBase, + threads: updateThread(nextBase.threads, event.payload.threadId, { + rollbackStatus: + event.payload.status === null + ? null + : { state: event.payload.status, updatedAt: event.payload.updatedAt }, + updatedAt: event.payload.updatedAt, + }), + }); + case "thread.reverted": return decodeForEvent(ThreadRevertedPayload, event.payload, event.type, "payload").pipe( Effect.map((payload) => { @@ -780,6 +792,7 @@ export function projectEvent( proposedPlans, activities, latestTurn, + rollbackStatus: null, updatedAt: event.occurredAt, }), }; diff --git a/apps/server/src/orchestration/runtimeLayer.ts b/apps/server/src/orchestration/runtimeLayer.ts index 779042e2f..07b82b28a 100644 --- a/apps/server/src/orchestration/runtimeLayer.ts +++ b/apps/server/src/orchestration/runtimeLayer.ts @@ -2,6 +2,7 @@ import * as Layer from "effect/Layer"; import { OrchestrationCommandReceiptRepositoryLive } from "../persistence/Layers/OrchestrationCommandReceipts.ts"; import { OrchestrationEventStoreLive } from "../persistence/Layers/OrchestrationEventStore.ts"; +import { RollbackSagaRepositoryLive } from "../persistence/Layers/RollbackSagas.ts"; import { OrchestrationEngineLive } from "./Layers/OrchestrationEngine.ts"; import { OrchestrationProjectionPipelineLive } from "./Layers/ProjectionPipeline.ts"; import { OrchestrationProjectionSnapshotQueryLive } from "./Layers/ProjectionSnapshotQuery.ts"; @@ -11,6 +12,7 @@ import * as ThreadPlanProgress from "./ThreadPlanProgress.ts"; export const OrchestrationEventInfrastructureLayerLive = Layer.mergeAll( OrchestrationEventStoreLive, OrchestrationCommandReceiptRepositoryLive, + RollbackSagaRepositoryLive, ); export const OrchestrationProjectionPipelineLayerLive = OrchestrationProjectionPipelineLive.pipe( diff --git a/apps/server/src/persistence/Layers/ProjectionThreads.ts b/apps/server/src/persistence/Layers/ProjectionThreads.ts index a43abfb86..dddd97f4c 100644 --- a/apps/server/src/persistence/Layers/ProjectionThreads.ts +++ b/apps/server/src/persistence/Layers/ProjectionThreads.ts @@ -42,6 +42,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { worktree_path, linked_pull_request_json, latest_turn_id, + rollback_status, + rollback_updated_at, created_at, updated_at, archived_at, @@ -72,6 +74,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { ${row.worktreePath}, ${row.linkedPullRequest === undefined || row.linkedPullRequest === null ? null : JSON.stringify(row.linkedPullRequest)}, ${row.latestTurnId}, + ${row.rollbackStatus ?? null}, + ${row.rollbackUpdatedAt ?? null}, ${row.createdAt}, ${row.updatedAt}, ${row.archivedAt}, @@ -102,6 +106,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { worktree_path = excluded.worktree_path, linked_pull_request_json = excluded.linked_pull_request_json, latest_turn_id = excluded.latest_turn_id, + rollback_status = excluded.rollback_status, + rollback_updated_at = excluded.rollback_updated_at, created_at = excluded.created_at, updated_at = excluded.updated_at, archived_at = excluded.archived_at, @@ -139,6 +145,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", @@ -178,6 +186,8 @@ const makeProjectionThreadRepository = Effect.gen(function* () { worktree_path AS "worktreePath", linked_pull_request_json AS "linkedPullRequest", latest_turn_id AS "latestTurnId", + rollback_status AS "rollbackStatus", + rollback_updated_at AS "rollbackUpdatedAt", created_at AS "createdAt", updated_at AS "updatedAt", archived_at AS "archivedAt", diff --git a/apps/server/src/persistence/Layers/RollbackSagas.test.ts b/apps/server/src/persistence/Layers/RollbackSagas.test.ts new file mode 100644 index 000000000..28b402adb --- /dev/null +++ b/apps/server/src/persistence/Layers/RollbackSagas.test.ts @@ -0,0 +1,254 @@ +// @effect-diagnostics preferSchemaOverJson:off +import { + CheckpointRef, + ProjectId, + ProviderInstanceId, + RuntimeSessionId, + ThreadId, +} from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; + +import { RollbackSagaRepository, type RollbackSagaState } from "../Services/RollbackSagas.ts"; +import { RollbackSagaRepositoryLive } from "./RollbackSagas.ts"; +import { SqlitePersistenceMemory } from "./Sqlite.ts"; + +const now = "2026-08-31T00:00:00.000Z"; +const threadA = ThreadId.make("thread-rollback-a"); +const threadB = ThreadId.make("thread-rollback-b"); +const threadC = ThreadId.make("thread-rollback-c"); +const projectId = ProjectId.make("project-rollback"); +const providerInstanceId = ProviderInstanceId.make("fake-absolute"); +const sessionIncarnationId = RuntimeSessionId.make("session-incarnation-1"); + +const makeState = ( + operationId: string, + threadId = threadA, + workspaceKey = "workspace-key", +): RollbackSagaState => ({ + operationId, + requestEventId: `event-${operationId}`, + threadId, + projectId, + workspaceKey, + workspaceCwd: "/private/workspace/canary", + sourceRevision: 2, + targetRevision: 1, + sourceCheckpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-rollback-a/turn/2"), + sourceCheckpointOid: "a".repeat(40), + targetCheckpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-rollback-a/turn/1"), + targetCheckpointOid: "b".repeat(40), + targetCheckpointDigest: "c".repeat(40), + providerInstanceId, + sessionIncarnationId, + phase: "source-anchor-capture-started", + attempt: 0, + lastErrorCode: null, + compensation: "none", + cleanup: "pending", + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: { leafId: "PRIVATE_LEAF_TARGET" }, + desiredAnchorDigest: "target-digest", + preimage: null, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: now, + updatedAt: now, +}); + +const layer = it.layer( + RollbackSagaRepositoryLive.pipe(Layer.provideMerge(SqlitePersistenceMemory)), +); + +layer("RollbackSagaRepository", (it) => { + it.effect("admits the saga and workspace lease atomically", () => + Effect.gen(function* () { + const repository = yield* RollbackSagaRepository; + yield* repository.admit(makeState("operation-a")); + + const competing = yield* repository + .admit(makeState("operation-b", threadB, "workspace-key")) + .pipe(Effect.result); + assert.equal(competing._tag, "Failure"); + assert.isTrue(Option.isNone(yield* repository.get("operation-b"))); + + const lease = yield* repository.findLeaseByWorkspace("workspace-key"); + assert.isTrue(Option.isSome(lease)); + if (Option.isSome(lease)) { + assert.equal(lease.value.operationId, "operation-a"); + assert.equal(lease.value.threadId, threadA); + } + assert.equal((yield* repository.listNonterminal()).length, 1); + }), + ); + + it.effect( + "uses owner and version CAS, clears stale startup owners, and releases the lease last", + () => + Effect.gen(function* () { + const repository = yield* RollbackSagaRepository; + const initial = makeState("operation-c", threadC, "workspace-c"); + yield* repository.admit(initial); + + const firstOwner = yield* repository.claim(initial.operationId, "owner-a"); + assert.isTrue(Option.isSome(firstOwner)); + if (Option.isNone(firstOwner)) return; + assert.equal(firstOwner.value.version, 0); + + assert.isTrue(Option.isNone(yield* repository.claim(initial.operationId, "owner-b"))); + assert.isTrue( + Option.isNone( + yield* repository.updateOwned({ + operationId: initial.operationId, + ownerId: "owner-a", + expectedVersion: 99, + state: { ...initial, phase: "source-anchor-captured" }, + }), + ), + ); + + const updated = yield* repository.updateOwned({ + operationId: initial.operationId, + ownerId: "owner-a", + expectedVersion: firstOwner.value.version, + state: { ...initial, phase: "source-anchor-captured" }, + }); + assert.isTrue(Option.isSome(updated)); + if (Option.isNone(updated)) return; + assert.equal(updated.value.version, 1); + + yield* repository.clearOwnersForStartup(); + const reclaimed = yield* repository.claim(initial.operationId, "owner-after-restart"); + assert.isTrue(Option.isSome(reclaimed)); + if (Option.isNone(reclaimed)) return; + const terminalState = { + ...reclaimed.value.state, + phase: "complete" as const, + cleanup: "complete" as const, + desiredAnchor: null, + desiredAnchorDigest: null, + }; + const released = yield* repository.releaseLeaseOwned({ + operationId: initial.operationId, + ownerId: "owner-after-restart", + expectedVersion: reclaimed.value.version, + state: terminalState, + }); + assert.isTrue(Option.isSome(released)); + assert.isTrue(Option.isNone(yield* repository.findLeaseByWorkspace("workspace-c"))); + assert.isFalse( + (yield* repository.listNonterminal()).some( + (record) => record.operationId === initial.operationId, + ), + ); + }), + ); + + it.effect( + "keeps exact provider anchors private and deletes only anchors newer than the target", + () => + Effect.gen(function* () { + const repository = yield* RollbackSagaRepository; + const sql = yield* SqlClient.SqlClient; + const privateCanary = "PRIVATE_PROVIDER_LEAF_CANARY"; + for (const checkpointTurnCount of [0, 1, 2]) { + yield* repository.putCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount, + providerInstanceId, + sessionIncarnationId, + checkpointRef: CheckpointRef.make( + `refs/t3/checkpoints/thread-rollback-a/turn/${checkpointTurnCount}`, + ), + checkpointOid: String(checkpointTurnCount).repeat(40), + anchor: { leafId: `${privateCanary}-${checkpointTurnCount}` }, + anchorDigest: `digest-${checkpointTurnCount}`, + capturedAt: now, + }); + } + + yield* repository.putCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 1, + providerInstanceId, + sessionIncarnationId, + checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-rollback-a/turn/1"), + checkpointOid: "1".repeat(40), + anchor: { leafId: `${privateCanary}-1` }, + anchorDigest: "digest-1", + capturedAt: "2026-08-31T01:00:00.000Z", + }); + + const conflicting = yield* repository + .putCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 1, + providerInstanceId, + sessionIncarnationId, + checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-rollback-a/turn/1"), + checkpointOid: "9".repeat(40), + anchor: { leafId: "MUST_NOT_OVERWRITE" }, + anchorDigest: "conflicting-digest", + capturedAt: now, + }) + .pipe(Effect.result); + assert.equal(conflicting._tag, "Failure"); + + const exact = yield* repository.getCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 1, + providerInstanceId, + sessionIncarnationId, + }); + assert.isTrue(Option.isSome(exact)); + if (Option.isSome(exact)) { + assert.deepEqual(exact.value.anchor, { leafId: `${privateCanary}-1` }); + } + const privateRows = yield* sql<{ readonly anchorJson: string }>` + SELECT anchor_json AS "anchorJson" FROM rollback_checkpoint_anchors + `; + assert.equal(privateRows.length, 3); + assert.isTrue(privateRows.every((row) => row.anchorJson.includes(privateCanary))); + + yield* repository.deleteCheckpointAnchorsAfter({ + threadId: threadA, + checkpointTurnCount: 1, + }); + assert.isTrue( + Option.isSome( + yield* repository.getCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 0, + providerInstanceId, + sessionIncarnationId, + }), + ), + ); + assert.isTrue( + Option.isSome( + yield* repository.getCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 1, + providerInstanceId, + sessionIncarnationId, + }), + ), + ); + assert.isTrue( + Option.isNone( + yield* repository.getCheckpointAnchor({ + threadId: threadA, + checkpointTurnCount: 2, + providerInstanceId, + sessionIncarnationId, + }), + ), + ); + }), + ); +}); diff --git a/apps/server/src/persistence/Layers/RollbackSagas.ts b/apps/server/src/persistence/Layers/RollbackSagas.ts new file mode 100644 index 000000000..d759ca52a --- /dev/null +++ b/apps/server/src/persistence/Layers/RollbackSagas.ts @@ -0,0 +1,404 @@ +// @effect-diagnostics preferSchemaOverJson:off +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import type { SqlError } from "effect/unstable/sql/SqlError"; +import { NonNegativeInt, ProjectId, ThreadId } from "@t3tools/contracts"; +import { + PersistenceDecodeError, + toPersistenceDecodeError, + toPersistenceSqlError, +} from "../Errors.ts"; +import { + RollbackCheckpointAnchor, + RollbackSagaRecord, + RollbackSagaRepository, + RollbackSagaState, + type RollbackSagaRepositoryShape, +} from "../Services/RollbackSagas.ts"; + +const SagaDbRow = Schema.Struct({ + operationId: Schema.String, + requestEventId: Schema.String, + threadId: ThreadId, + projectId: ProjectId, + workspaceKey: Schema.String, + phase: Schema.String, + terminal: Schema.Number, + ownerId: Schema.NullOr(Schema.String), + version: NonNegativeInt, + privateStateJson: Schema.String, + createdAt: Schema.String, + updatedAt: Schema.String, +}); +const LeaseDbRow = Schema.Struct({ + operationId: Schema.String, + threadId: ThreadId, + projectId: ProjectId, +}); +const AnchorDbRow = Schema.Struct({ + threadId: ThreadId, + checkpointTurnCount: NonNegativeInt, + providerInstanceId: Schema.String, + sessionIncarnationId: Schema.String, + checkpointRef: Schema.String, + checkpointOid: Schema.String, + anchorJson: Schema.String, + anchorDigest: Schema.String, + capturedAt: Schema.String, +}); +const decodeSagaDbRows = Schema.decodeUnknownEffect(Schema.Array(SagaDbRow)); +const decodeLeaseDbRows = Schema.decodeUnknownEffect(Schema.Array(LeaseDbRow)); +const decodeAnchorDbRows = Schema.decodeUnknownEffect(Schema.Array(AnchorDbRow)); +const decodeSagaState = Schema.decodeUnknownEffect(RollbackSagaState); +const decodeSagaRecord = Schema.decodeUnknownEffect(RollbackSagaRecord); +const decodeAnchor = Schema.decodeUnknownEffect(RollbackCheckpointAnchor); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + let nonterminalFenceCache: ReadonlyArray | null = null; + + const mapSagaRow = Effect.fn("RollbackSagaRepository.mapSagaRow")(function* ( + row: typeof SagaDbRow.Type, + ) { + const state = yield* Effect.try({ + try: () => JSON.parse(row.privateStateJson) as unknown, + catch: (cause) => + toPersistenceDecodeError("RollbackSagaRepository.parseSagaState")( + cause as Schema.SchemaError, + ), + }).pipe( + Effect.flatMap(decodeSagaState), + Effect.mapError((cause) => + cause._tag === "PersistenceDecodeError" + ? cause + : toPersistenceDecodeError("RollbackSagaRepository.decodeSagaState")(cause), + ), + ); + return yield* decodeSagaRecord({ + operationId: row.operationId, + requestEventId: row.requestEventId, + threadId: row.threadId, + projectId: row.projectId, + workspaceKey: row.workspaceKey, + phase: row.phase, + terminal: row.terminal === 1, + ownerId: row.ownerId, + version: row.version, + state, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }).pipe(Effect.mapError(toPersistenceDecodeError("RollbackSagaRepository.decodeSagaRecord"))); + }); + + const selectSagaRows = (query: Effect.Effect, SqlError>) => + query.pipe( + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.select")), + Effect.flatMap(decodeSagaDbRows), + Effect.mapError((cause) => + cause._tag === "PersistenceSqlError" + ? cause + : toPersistenceDecodeError("RollbackSagaRepository.decodeRows")(cause), + ), + Effect.flatMap((rows) => Effect.forEach(rows, mapSagaRow)), + ); + + const admit: RollbackSagaRepositoryShape["admit"] = (state) => + sql + .withTransaction( + Effect.gen(function* () { + yield* sql` + INSERT INTO rollback_sagas ( + operation_id, request_event_id, thread_id, project_id, workspace_key, + phase, terminal, owner_id, version, private_state_json, created_at, updated_at + ) VALUES ( + ${state.operationId}, ${state.requestEventId}, ${state.threadId}, ${state.projectId}, + ${state.workspaceKey}, ${state.phase}, 0, NULL, 0, ${JSON.stringify(state)}, + ${state.createdAt}, ${state.updatedAt} + ) + `; + yield* sql` + INSERT INTO rollback_workspace_leases ( + workspace_key, operation_id, thread_id, project_id, acquired_at + ) VALUES ( + ${state.workspaceKey}, ${state.operationId}, ${state.threadId}, ${state.projectId}, + ${state.createdAt} + ) + `; + }), + ) + .pipe( + Effect.tap(() => + Effect.sync(() => { + nonterminalFenceCache = null; + }), + ), + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.admit")), + ); + + const get: RollbackSagaRepositoryShape["get"] = (operationId) => + selectSagaRows(sql` + SELECT operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + FROM rollback_sagas WHERE operation_id = ${operationId} LIMIT 1 + `).pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))); + + const getByRequestEvent: RollbackSagaRepositoryShape["getByRequestEvent"] = (requestEventId) => + selectSagaRows(sql` + SELECT operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + FROM rollback_sagas WHERE request_event_id = ${requestEventId} LIMIT 1 + `).pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))); + + const getActiveByThread: RollbackSagaRepositoryShape["getActiveByThread"] = (threadId) => + selectSagaRows(sql` + SELECT operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + FROM rollback_sagas WHERE thread_id = ${threadId} AND terminal = 0 LIMIT 1 + `).pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))); + + const listNonterminal: RollbackSagaRepositoryShape["listNonterminal"] = () => + selectSagaRows(sql` + SELECT operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + FROM rollback_sagas WHERE terminal = 0 ORDER BY created_at ASC, operation_id ASC + `).pipe( + Effect.tap((records) => + Effect.sync(() => { + nonterminalFenceCache = records; + }), + ), + ); + + const listNonterminalForFence: RollbackSagaRepositoryShape["listNonterminalForFence"] = () => + nonterminalFenceCache === null ? listNonterminal() : Effect.succeed(nonterminalFenceCache); + + const clearOwnersForStartup: RollbackSagaRepositoryShape["clearOwnersForStartup"] = () => + sql`UPDATE rollback_sagas SET owner_id = NULL WHERE terminal = 0`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.clearOwnersForStartup")), + ); + + const claim: RollbackSagaRepositoryShape["claim"] = (operationId, ownerId) => + selectSagaRows(sql` + UPDATE rollback_sagas + SET owner_id = ${ownerId} + WHERE operation_id = ${operationId} AND terminal = 0 AND (owner_id IS NULL OR owner_id = ${ownerId}) + RETURNING operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + `).pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))); + + const updateOwned: RollbackSagaRepositoryShape["updateOwned"] = (input) => + selectSagaRows(sql` + UPDATE rollback_sagas + SET phase = ${input.state.phase}, terminal = ${input.terminal === true ? 1 : 0}, + private_state_json = ${JSON.stringify(input.state)}, updated_at = ${input.state.updatedAt}, + version = version + 1 + WHERE operation_id = ${input.operationId} AND owner_id = ${input.ownerId} + AND version = ${input.expectedVersion} AND terminal = 0 + RETURNING operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + `).pipe( + Effect.map((rows) => Option.fromNullishOr(rows[0])), + Effect.tap((updated) => + input.terminal === true && Option.isSome(updated) + ? Effect.sync(() => { + nonterminalFenceCache = null; + }) + : Effect.void, + ), + ); + + const releaseOwnerOwned: RollbackSagaRepositoryShape["releaseOwnerOwned"] = ( + operationId, + ownerId, + ) => + sql`UPDATE rollback_sagas SET owner_id = NULL WHERE operation_id = ${operationId} AND owner_id = ${ownerId} AND terminal = 0`.pipe( + Effect.asVoid, + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.releaseOwnerOwned")), + ); + + const releaseLeaseOwned: RollbackSagaRepositoryShape["releaseLeaseOwned"] = (input) => + sql + .withTransaction( + Effect.gen(function* () { + const rows = yield* selectSagaRows(sql` + UPDATE rollback_sagas + SET phase = ${input.state.phase}, terminal = 1, owner_id = NULL, + private_state_json = ${JSON.stringify(input.state)}, updated_at = ${input.state.updatedAt}, + version = version + 1 + WHERE operation_id = ${input.operationId} AND owner_id = ${input.ownerId} + AND version = ${input.expectedVersion} AND terminal = 0 + RETURNING operation_id AS "operationId", request_event_id AS "requestEventId", + thread_id AS "threadId", project_id AS "projectId", workspace_key AS "workspaceKey", + phase, terminal, owner_id AS "ownerId", version, + private_state_json AS "privateStateJson", created_at AS "createdAt", updated_at AS "updatedAt" + `); + if (rows.length === 0) return Option.none(); + yield* sql`DELETE FROM rollback_workspace_leases WHERE operation_id = ${input.operationId}`; + return Option.some(rows[0]!); + }), + ) + .pipe( + Effect.tap((released) => + Option.isSome(released) + ? Effect.sync(() => { + nonterminalFenceCache = null; + }) + : Effect.void, + ), + Effect.mapError((cause) => + cause._tag === "PersistenceDecodeError" + ? cause + : toPersistenceSqlError("RollbackSagaRepository.releaseLeaseOwned")(cause), + ), + ); + + const findLeaseByWorkspace: RollbackSagaRepositoryShape["findLeaseByWorkspace"] = ( + workspaceKey, + ) => + sql` + SELECT operation_id AS "operationId", thread_id AS "threadId", project_id AS "projectId" + FROM rollback_workspace_leases WHERE workspace_key = ${workspaceKey} LIMIT 1 + `.pipe( + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.findLeaseByWorkspace")), + Effect.flatMap(decodeLeaseDbRows), + Effect.mapError((cause) => + cause._tag === "PersistenceSqlError" + ? cause + : toPersistenceDecodeError("RollbackSagaRepository.decodeLease")(cause), + ), + Effect.map((rows) => Option.fromNullishOr(rows[0])), + ); + + const putCheckpointAnchor: RollbackSagaRepositoryShape["putCheckpointAnchor"] = Effect.fn( + "RollbackSagaRepository.putCheckpointAnchor", + )(function* (anchor) { + const anchorJson = yield* Effect.try({ + try: () => JSON.stringify(anchor.anchor), + catch: (cause) => + new PersistenceDecodeError({ + operation: "RollbackSagaRepository.encodeCheckpointAnchor", + issue: "JsonEncoding", + cause, + }), + }); + const rows = yield* sql<{ readonly anchorDigest: string }>` + INSERT INTO rollback_checkpoint_anchors ( + thread_id, checkpoint_turn_count, provider_instance_id, session_incarnation_id, + checkpoint_ref, checkpoint_oid, anchor_json, anchor_digest, captured_at + ) VALUES ( + ${anchor.threadId}, ${anchor.checkpointTurnCount}, ${anchor.providerInstanceId}, + ${anchor.sessionIncarnationId}, ${anchor.checkpointRef}, ${anchor.checkpointOid}, + ${anchorJson}, ${anchor.anchorDigest}, ${anchor.capturedAt} + ) + ON CONFLICT (thread_id, checkpoint_turn_count, provider_instance_id, session_incarnation_id) + DO UPDATE SET captured_at = rollback_checkpoint_anchors.captured_at + WHERE rollback_checkpoint_anchors.checkpoint_ref = excluded.checkpoint_ref + AND rollback_checkpoint_anchors.checkpoint_oid = excluded.checkpoint_oid + AND rollback_checkpoint_anchors.anchor_digest = excluded.anchor_digest + RETURNING anchor_digest AS "anchorDigest" + `.pipe(Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.putCheckpointAnchor"))); + if (rows.length !== 1) { + return yield* new PersistenceDecodeError({ + operation: "RollbackSagaRepository.immutableCheckpointAnchorConflict", + issue: "ImmutableIdentityConflict", + }); + } + }); + + const getCheckpointAnchor: RollbackSagaRepositoryShape["getCheckpointAnchor"] = (input) => + sql` + SELECT thread_id AS "threadId", checkpoint_turn_count AS "checkpointTurnCount", + provider_instance_id AS "providerInstanceId", session_incarnation_id AS "sessionIncarnationId", + checkpoint_ref AS "checkpointRef", checkpoint_oid AS "checkpointOid", + anchor_json AS "anchorJson", anchor_digest AS "anchorDigest", captured_at AS "capturedAt" + FROM rollback_checkpoint_anchors + WHERE thread_id = ${input.threadId} AND checkpoint_turn_count = ${input.checkpointTurnCount} + AND provider_instance_id = ${input.providerInstanceId} + AND session_incarnation_id = ${input.sessionIncarnationId} + LIMIT 1 + `.pipe( + Effect.mapError(toPersistenceSqlError("RollbackSagaRepository.getCheckpointAnchor")), + Effect.flatMap(decodeAnchorDbRows), + Effect.mapError((cause) => + cause._tag === "PersistenceSqlError" + ? cause + : toPersistenceDecodeError("RollbackSagaRepository.decodeAnchorRows")(cause), + ), + Effect.flatMap((rows) => + Effect.forEach(rows, (row) => + Effect.try({ + try: () => JSON.parse(row.anchorJson) as unknown, + catch: (cause) => + toPersistenceDecodeError("RollbackSagaRepository.parseAnchor")( + cause as Schema.SchemaError, + ), + }).pipe( + Effect.flatMap((anchor) => + decodeAnchor({ + threadId: row.threadId, + checkpointTurnCount: row.checkpointTurnCount, + providerInstanceId: row.providerInstanceId, + sessionIncarnationId: row.sessionIncarnationId, + checkpointRef: row.checkpointRef, + checkpointOid: row.checkpointOid, + anchor, + anchorDigest: row.anchorDigest, + capturedAt: row.capturedAt, + }), + ), + Effect.mapError((cause) => + cause._tag === "PersistenceDecodeError" + ? cause + : toPersistenceDecodeError("RollbackSagaRepository.decodeAnchor")(cause), + ), + ), + ), + ), + Effect.map((rows) => Option.fromNullishOr(rows[0])), + ); + + const deleteCheckpointAnchorsAfter: RollbackSagaRepositoryShape["deleteCheckpointAnchorsAfter"] = + (input) => + sql`DELETE FROM rollback_checkpoint_anchors WHERE thread_id = ${input.threadId} AND checkpoint_turn_count > ${input.checkpointTurnCount}`.pipe( + Effect.asVoid, + Effect.mapError( + toPersistenceSqlError("RollbackSagaRepository.deleteCheckpointAnchorsAfter"), + ), + ); + + return RollbackSagaRepository.of({ + admit, + get, + getByRequestEvent, + getActiveByThread, + listNonterminal, + listNonterminalForFence, + clearOwnersForStartup, + claim, + updateOwned, + releaseOwnerOwned, + releaseLeaseOwned, + findLeaseByWorkspace, + putCheckpointAnchor, + getCheckpointAnchor, + deleteCheckpointAnchorsAfter, + }); +}); + +export const RollbackSagaRepositoryLive = Layer.effect(RollbackSagaRepository, make); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 341e53616..cc285c327 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -62,6 +62,7 @@ import Migration0047 from "./Migrations/047_ProjectionThreadsUnsettledAt.ts"; import Migration0048 from "./Migrations/048_ProjectionThreadSessionPendingTurnRequest.ts"; import Migration0049 from "./Migrations/049_ProjectionThreadSessionPendingStop.ts"; import Migration0050 from "./Migrations/050_PrimeAgentRecoveryLedger.ts"; +import Migration0051 from "./Migrations/051_DurableRollbackSagas.ts"; /** * Migration loader with all migrations defined inline. * @@ -146,6 +147,7 @@ export const migrationEntries = [ [48, "ProjectionThreadSessionPendingTurnRequest", Migration0048], [49, "ProjectionThreadSessionPendingStop", Migration0049], [50, "PrimeAgentRecoveryLedger", Migration0050], + [51, "DurableRollbackSagas", Migration0051], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/051_DurableRollbackSagas.ts b/apps/server/src/persistence/Migrations/051_DurableRollbackSagas.ts new file mode 100644 index 000000000..08fc1bf76 --- /dev/null +++ b/apps/server/src/persistence/Migrations/051_DurableRollbackSagas.ts @@ -0,0 +1,74 @@ +import * as SqlClient from "effect/unstable/sql/SqlClient"; +import * as Effect from "effect/Effect"; + +/** Private rollback state. Nothing in these tables is projected or sent to clients. */ +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + const projectionColumns = yield* sql<{ + readonly name: string; + }>`PRAGMA table_info(projection_threads)`; + const projectionColumnNames = new Set(projectionColumns.map((column) => column.name)); + if (!projectionColumnNames.has("rollback_status")) { + yield* sql.unsafe("ALTER TABLE projection_threads ADD COLUMN rollback_status TEXT"); + } + if (!projectionColumnNames.has("rollback_updated_at")) { + yield* sql.unsafe("ALTER TABLE projection_threads ADD COLUMN rollback_updated_at TEXT"); + } + + yield* sql` + CREATE TABLE IF NOT EXISTS rollback_sagas ( + operation_id TEXT PRIMARY KEY, + request_event_id TEXT NOT NULL UNIQUE, + thread_id TEXT NOT NULL, + project_id TEXT NOT NULL, + workspace_key TEXT NOT NULL, + phase TEXT NOT NULL, + terminal INTEGER NOT NULL DEFAULT 0, + owner_id TEXT, + version INTEGER NOT NULL DEFAULT 0, + private_state_json TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + `; + yield* sql` + CREATE UNIQUE INDEX IF NOT EXISTS idx_rollback_sagas_active_thread + ON rollback_sagas(thread_id) WHERE terminal = 0 + `; + yield* sql` + CREATE INDEX IF NOT EXISTS idx_rollback_sagas_nonterminal + ON rollback_sagas(terminal, created_at, operation_id) + `; + + yield* sql` + CREATE TABLE IF NOT EXISTS rollback_workspace_leases ( + workspace_key TEXT PRIMARY KEY, + operation_id TEXT NOT NULL UNIQUE, + thread_id TEXT NOT NULL, + project_id TEXT NOT NULL, + acquired_at TEXT NOT NULL, + FOREIGN KEY (operation_id) REFERENCES rollback_sagas(operation_id) ON DELETE CASCADE + ) + `; + + yield* sql` + CREATE TABLE IF NOT EXISTS rollback_checkpoint_anchors ( + thread_id TEXT NOT NULL, + checkpoint_turn_count INTEGER NOT NULL, + provider_instance_id TEXT NOT NULL, + session_incarnation_id TEXT NOT NULL, + checkpoint_ref TEXT NOT NULL, + checkpoint_oid TEXT NOT NULL, + anchor_json TEXT NOT NULL, + anchor_digest TEXT NOT NULL, + captured_at TEXT NOT NULL, + PRIMARY KEY ( + thread_id, + checkpoint_turn_count, + provider_instance_id, + session_incarnation_id + ) + ) + `; +}); diff --git a/apps/server/src/persistence/Services/ProjectionThreads.ts b/apps/server/src/persistence/Services/ProjectionThreads.ts index daa1137ae..d5ab5a0f0 100644 --- a/apps/server/src/persistence/Services/ProjectionThreads.ts +++ b/apps/server/src/persistence/Services/ProjectionThreads.ts @@ -11,6 +11,7 @@ import { IsoDateTime, ModelSelection, NonNegativeInt, + OrchestrationRollbackPublicState, ProjectId, ProviderInteractionMode, RuntimeMode, @@ -36,6 +37,8 @@ export const ProjectionThread = Schema.Struct({ worktreePath: Schema.NullOr(Schema.String), linkedPullRequest: Schema.optional(Schema.NullOr(ThreadLinkedPullRequest)), latestTurnId: Schema.NullOr(TurnId), + rollbackStatus: Schema.optional(Schema.NullOr(OrchestrationRollbackPublicState)), + rollbackUpdatedAt: Schema.optional(Schema.NullOr(IsoDateTime)), createdAt: IsoDateTime, updatedAt: IsoDateTime, archivedAt: Schema.NullOr(IsoDateTime), diff --git a/apps/server/src/persistence/Services/RollbackSagas.ts b/apps/server/src/persistence/Services/RollbackSagas.ts new file mode 100644 index 000000000..ab494b5ad --- /dev/null +++ b/apps/server/src/persistence/Services/RollbackSagas.ts @@ -0,0 +1,170 @@ +import * as Context from "effect/Context"; +import type * as Effect from "effect/Effect"; +import type * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { + CheckpointRef, + IsoDateTime, + NonNegativeInt, + ProjectId, + ProviderInstanceId, + RuntimeSessionId, + ThreadId, +} from "@t3tools/contracts"; +import type { PersistenceDecodeError, PersistenceSqlError } from "../Errors.ts"; + +export const RollbackSagaPhase = Schema.Literals([ + "source-anchor-capture-started", + "source-anchor-captured", + "preimage-capture-started", + "preimage-captured", + "workspace-apply-started", + "workspace-applied", + "provider-apply-started", + "provider-applied", + "projection-commit-started", + "projection-committed", + "cleanup-started", + "compensation-workspace-started", + "compensation-workspace-complete", + "compensation-provider-started", + "compensated", + "manual-recovery", + "complete", +]); +export type RollbackSagaPhase = typeof RollbackSagaPhase.Type; + +export const RollbackSagaState = Schema.Struct({ + operationId: Schema.String, + requestEventId: Schema.String, + threadId: ThreadId, + projectId: ProjectId, + workspaceKey: Schema.String, + workspaceCwd: Schema.String, + sourceRevision: NonNegativeInt, + targetRevision: NonNegativeInt, + sourceCheckpointRef: CheckpointRef, + sourceCheckpointOid: Schema.String, + targetCheckpointRef: CheckpointRef, + targetCheckpointOid: Schema.String, + targetCheckpointDigest: Schema.String, + providerInstanceId: ProviderInstanceId, + sessionIncarnationId: RuntimeSessionId, + phase: RollbackSagaPhase, + attempt: NonNegativeInt, + lastErrorCode: Schema.NullOr(Schema.String), + compensation: Schema.Literals(["none", "required", "workspace", "provider", "proved", "manual"]), + cleanup: Schema.Literals(["pending", "running", "complete"]), + sourceAnchor: Schema.NullOr(Schema.Json), + sourceAnchorDigest: Schema.NullOr(Schema.String), + desiredAnchor: Schema.NullOr(Schema.Json), + desiredAnchorDigest: Schema.NullOr(Schema.String), + preimage: Schema.NullOr(Schema.Unknown), + workspaceReceiptDigest: Schema.NullOr(Schema.String), + providerReceiptDigest: Schema.NullOr(Schema.String), + projectionCommitSequence: Schema.NullOr(NonNegativeInt), + createdAt: IsoDateTime, + updatedAt: IsoDateTime, +}); +export type RollbackSagaState = typeof RollbackSagaState.Type; + +export const RollbackSagaRecord = Schema.Struct({ + operationId: Schema.String, + requestEventId: Schema.String, + threadId: ThreadId, + projectId: ProjectId, + workspaceKey: Schema.String, + phase: RollbackSagaPhase, + terminal: Schema.Boolean, + ownerId: Schema.NullOr(Schema.String), + version: NonNegativeInt, + state: RollbackSagaState, + createdAt: IsoDateTime, + updatedAt: IsoDateTime, +}); +export type RollbackSagaRecord = typeof RollbackSagaRecord.Type; + +export const RollbackCheckpointAnchor = Schema.Struct({ + threadId: ThreadId, + checkpointTurnCount: NonNegativeInt, + providerInstanceId: ProviderInstanceId, + sessionIncarnationId: RuntimeSessionId, + checkpointRef: CheckpointRef, + checkpointOid: Schema.String, + anchor: Schema.Json, + anchorDigest: Schema.String, + capturedAt: IsoDateTime, +}); +export type RollbackCheckpointAnchor = typeof RollbackCheckpointAnchor.Type; + +export type RollbackSagaRepositoryError = PersistenceSqlError | PersistenceDecodeError; + +export interface RollbackSagaRepositoryShape { + readonly admit: (state: RollbackSagaState) => Effect.Effect; + readonly get: ( + operationId: string, + ) => Effect.Effect, RollbackSagaRepositoryError>; + readonly getByRequestEvent: ( + requestEventId: string, + ) => Effect.Effect, RollbackSagaRepositoryError>; + readonly getActiveByThread: ( + threadId: ThreadId, + ) => Effect.Effect, RollbackSagaRepositoryError>; + readonly listNonterminal: () => Effect.Effect< + ReadonlyArray, + RollbackSagaRepositoryError + >; + /** Process-local hot-path view. The repository invalidates it on every admission/terminal CAS. */ + readonly listNonterminalForFence: () => Effect.Effect< + ReadonlyArray, + RollbackSagaRepositoryError + >; + readonly clearOwnersForStartup: () => Effect.Effect; + readonly claim: ( + operationId: string, + ownerId: string, + ) => Effect.Effect, RollbackSagaRepositoryError>; + readonly updateOwned: (input: { + readonly operationId: string; + readonly ownerId: string; + readonly expectedVersion: number; + readonly state: RollbackSagaState; + readonly terminal?: boolean; + }) => Effect.Effect, RollbackSagaRepositoryError>; + readonly releaseOwnerOwned: ( + operationId: string, + ownerId: string, + ) => Effect.Effect; + readonly releaseLeaseOwned: (input: { + readonly operationId: string; + readonly ownerId: string; + readonly expectedVersion: number; + readonly state: RollbackSagaState; + }) => Effect.Effect, RollbackSagaRepositoryError>; + readonly findLeaseByWorkspace: (workspaceKey: string) => Effect.Effect< + Option.Option<{ + readonly operationId: string; + readonly threadId: ThreadId; + readonly projectId: ProjectId; + }>, + RollbackSagaRepositoryError + >; + readonly putCheckpointAnchor: ( + anchor: RollbackCheckpointAnchor, + ) => Effect.Effect; + readonly getCheckpointAnchor: (input: { + readonly threadId: ThreadId; + readonly checkpointTurnCount: number; + readonly providerInstanceId: ProviderInstanceId; + readonly sessionIncarnationId: RuntimeSessionId; + }) => Effect.Effect, RollbackSagaRepositoryError>; + readonly deleteCheckpointAnchorsAfter: (input: { + readonly threadId: ThreadId; + readonly checkpointTurnCount: number; + }) => Effect.Effect; +} + +export class RollbackSagaRepository extends Context.Service< + RollbackSagaRepository, + RollbackSagaRepositoryShape +>()("t3/persistence/Services/RollbackSagas/RollbackSagaRepository") {} diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index a4d7b7203..7e5c52315 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -95,6 +95,7 @@ import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; import * as ServerSettings from "../../serverSettings.ts"; +import { RollbackSagaRepository } from "../../persistence/Services/RollbackSagas.ts"; const isModelSelection = Schema.is(ModelSelection); /** @@ -314,6 +315,26 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const registry = yield* ProviderAdapterRegistry.ProviderAdapterRegistry; const directory = yield* ProviderSessionDirectory.ProviderSessionDirectory; + const rollbackRepository = yield* Effect.serviceOption(RollbackSagaRepository); + const assertNotRollbackFenced = Effect.fn("ProviderService.assertNotRollbackFenced")(function* ( + threadId: ThreadId, + operation: string, + ) { + if (Option.isNone(rollbackRepository)) return; + const active = yield* rollbackRepository.value + .getActiveByThread(threadId) + .pipe( + Effect.mapError(() => + toValidationError(operation, "Rollback mutation fence could not be verified."), + ), + ); + if (Option.isSome(active)) { + return yield* toValidationError( + operation, + "The provider session is fenced by an active rollback operation.", + ); + } + }); const serverSettings = yield* ServerSettings.ServerSettingsService; const issueMcpCredential = options?.issueMcpCredential ?? McpSessionRegistry.issueActiveMcpCredential; @@ -1364,6 +1385,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderSendTurnInput, payload: rawInput, }); + yield* assertNotRollbackFenced(parsed.threadId, "ProviderService.sendTurn"); const attachments = parsed.attachments ?? []; if (!parsed.input && attachments.length === 0) { @@ -1691,6 +1713,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderInterruptTurnInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.interruptTurn"); let metricProvider = "unknown"; return yield* Effect.gen(function* () { const routed = yield* resolveRoutableSession({ @@ -1773,6 +1796,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderRespondToUserInputInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.respondToUserInput"); let metricProvider = "unknown"; return yield* Effect.gen(function* () { const routed = yield* resolveRoutableSession({ @@ -1807,6 +1831,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderRespondToInteractionInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.respondToInteraction"); let metricProvider = "unknown"; return yield* Effect.gen(function* () { const routed = yield* resolveRoutableSession({ @@ -2128,6 +2153,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderFollowUpInput, payload: rawInput, }); + yield* assertNotRollbackFenced(parsed.threadId, "ProviderService.followUp"); const input = { ...parsed, attachments: parsed.attachments ?? [] }; if (!input.input && input.attachments.length === 0) { return yield* new ProviderValidationError({ @@ -2207,6 +2233,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderClearSessionInputQueueInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.clearSessionInputQueue"); const routed = yield* resolveRoutableSession({ threadId: input.threadId, operation: "ProviderService.clearSessionInputQueue", @@ -2238,6 +2265,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderRemoveOnlySessionInputQueueItemInput, payload: rawInput, }); + yield* assertNotRollbackFenced( + input.threadId, + "ProviderService.removeOnlySessionInputQueueItem", + ); const routed = yield* resolveRoutableSession({ threadId: input.threadId, operation: "ProviderService.removeOnlySessionInputQueueItem", @@ -2271,6 +2302,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderSetSessionInputQueueModeInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.setSessionInputQueueMode"); const routed = yield* resolveRoutableSession({ threadId: input.threadId, operation: "ProviderService.setSessionInputQueueMode", @@ -2465,6 +2497,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( schema: ProviderStopSessionInput, payload: rawInput, }); + yield* assertNotRollbackFenced(input.threadId, "ProviderService.stopSession"); let metricProvider = "unknown"; // Invalidate before the first directory read. A first-turn start can be // inside adapter creation without having a persisted binding yet; stop @@ -2851,6 +2884,86 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const getInstanceInfo: ProviderServiceMethod<"getInstanceInfo"> = (instanceId) => registry.getInstanceInfo(instanceId); + const resolveAbsoluteConversationRollback = Effect.fn( + "ProviderService.resolveAbsoluteConversationRollback", + )(function* (threadId: ThreadId) { + const routed = yield* resolveRoutableSession({ + threadId, + operation: "ProviderService.absoluteConversationRollback", + allowRecovery: true, + }); + const operations = routed.adapter.absoluteConversationRollback; + if ( + routed.adapter.capabilities.conversationRollback !== "absolute" || + operations === undefined + ) { + return yield* toValidationError( + "ProviderService.absoluteConversationRollback", + "Exact provider conversation rollback is unavailable.", + ); + } + return { routed, operations } as const; + }); + + const hasAbsoluteConversationRollback: NonNullable< + ProviderServiceMethod<"hasAbsoluteConversationRollback"> + > = Effect.fn("hasAbsoluteConversationRollback")(function* (threadId) { + const routed = yield* resolveRoutableSession({ + threadId, + operation: "ProviderService.hasAbsoluteConversationRollback", + allowRecovery: true, + }); + return ( + routed.adapter.capabilities.conversationRollback === "absolute" && + routed.adapter.absoluteConversationRollback !== undefined + ); + }); + + const captureConversationAnchor: NonNullable> = + Effect.fn("captureConversationAnchor")(function* (threadId) { + const { operations } = yield* resolveAbsoluteConversationRollback(threadId); + return yield* operations + .captureAnchor(threadId) + .pipe( + Effect.mapError(() => + toValidationError( + "ProviderService.captureConversationAnchor", + "The provider could not prove its current conversation anchor.", + ), + ), + ); + }); + + const inspectConversationAnchor: NonNullable> = + Effect.fn("inspectConversationAnchor")(function* (threadId) { + const { operations } = yield* resolveAbsoluteConversationRollback(threadId); + return yield* operations + .inspectAnchor(threadId) + .pipe( + Effect.mapError(() => + toValidationError( + "ProviderService.inspectConversationAnchor", + "The provider could not inspect its current conversation anchor.", + ), + ), + ); + }); + + const applyConversationAnchor: NonNullable> = + Effect.fn("applyConversationAnchor")(function* (input) { + const { operations } = yield* resolveAbsoluteConversationRollback(input.threadId); + return yield* operations + .applyAnchor(input.threadId, input.anchor) + .pipe( + Effect.mapError(() => + toValidationError( + "ProviderService.applyConversationAnchor", + "The provider did not prove an exact conversation anchor update.", + ), + ), + ); + }); + const rollbackConversation: ProviderServiceMethod<"rollbackConversation"> = Effect.fn( "rollbackConversation", )(function* (rawInput) { @@ -3096,6 +3209,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( getCapabilities, getInstanceInfo, rollbackConversation, + hasAbsoluteConversationRollback, + captureConversationAnchor, + inspectConversationAnchor, + applyConversationAnchor, uploadFeedback, // Each access creates a fresh PubSub subscription so that multiple // consumers (ProviderRuntimeIngestion, CheckpointReactor, etc.) each diff --git a/apps/server/src/provider/Services/ProviderAdapter.ts b/apps/server/src/provider/Services/ProviderAdapter.ts index c7317c686..d116bf7d2 100644 --- a/apps/server/src/provider/Services/ProviderAdapter.ts +++ b/apps/server/src/provider/Services/ProviderAdapter.ts @@ -42,6 +42,7 @@ import type { } from "@t3tools/contracts"; import type * as Effect from "effect/Effect"; import type * as Stream from "effect/Stream"; +import type { Json } from "effect/Schema"; import type { ProviderRuntimeFence } from "../ProviderDriver.ts"; export type ProviderSessionModelSwitchMode = "in-session" | "unsupported"; @@ -57,6 +58,23 @@ export const BUILT_IN_ADAPTER_CONVERSATION_ROLLBACK_MODES = { primeDaemon: "unsupported", } as const satisfies Record; +export interface ProviderConversationAnchorReceipt { + /** Provider-private absolute identity. It must never enter public events or logs. */ + readonly anchor: Json; + /** Provider-private stable digest used only for equality checks inside the saga. */ + readonly digest: string; +} + +export interface ProviderAbsoluteConversationRollback { + readonly captureAnchor: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly inspectAnchor: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly applyAnchor: (threadId: ThreadId, anchor: Json) => Effect.Effect; +} + export interface ProviderAdapterCapabilities { /** * Declares whether changing the model on an existing session is supported. @@ -88,6 +106,12 @@ export interface ProviderAdapterShape { /** Server-private materialization fence. It never crosses provider contracts. */ readonly runtimeFence?: ProviderRuntimeFence | undefined; + /** + * Exact provider conversation control. Production adapters intentionally omit + * this until they can apply and inspect an immutable absolute anchor. + */ + readonly absoluteConversationRollback?: ProviderAbsoluteConversationRollback; + /** * Start a provider-backed session. */ diff --git a/apps/server/src/provider/Services/ProviderService.ts b/apps/server/src/provider/Services/ProviderService.ts index f74f6cc2b..84cf4ae04 100644 --- a/apps/server/src/provider/Services/ProviderService.ts +++ b/apps/server/src/provider/Services/ProviderService.ts @@ -58,9 +58,13 @@ import type { import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; import type * as Stream from "effect/Stream"; +import type { Json } from "effect/Schema"; import type { ProviderServiceError } from "../Errors.ts"; -import type { ProviderAdapterCapabilities } from "./ProviderAdapter.ts"; +import type { + ProviderAdapterCapabilities, + ProviderConversationAnchorReceipt, +} from "./ProviderAdapter.ts"; import type { ProviderInstanceRoutingInfo } from "./ProviderAdapterRegistry.ts"; /** @@ -251,6 +255,21 @@ export interface ProviderServiceShape { readonly numTurns: number; }) => Effect.Effect; + /** Private exact-anchor operations used only by the durable rollback saga. */ + readonly hasAbsoluteConversationRollback?: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly captureConversationAnchor?: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly inspectConversationAnchor?: ( + threadId: ThreadId, + ) => Effect.Effect; + readonly applyConversationAnchor?: (input: { + readonly threadId: ThreadId; + readonly anchor: Json; + }) => Effect.Effect; + /** * Upload a thread and return the provider's shareable feedback identifier. */ diff --git a/apps/server/src/rollback/RollbackAdmission.test.ts b/apps/server/src/rollback/RollbackAdmission.test.ts new file mode 100644 index 000000000..4dbcef8b2 --- /dev/null +++ b/apps/server/src/rollback/RollbackAdmission.test.ts @@ -0,0 +1,271 @@ +import { + CheckpointRef, + ProjectId, + ProviderInstanceId, + RuntimeSessionId, + ThreadId, + TurnId, + type OrchestrationReadModel, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { checkpointRefForThreadTurn } from "../checkpointing/Utils.ts"; +import { + RollbackSagaRepository, + type RollbackSagaRepositoryShape, +} from "../persistence/Services/RollbackSagas.ts"; +import { ProviderService } from "../provider/Services/ProviderService.ts"; +import { make as makeRollbackAdmission } from "./RollbackAdmission.ts"; +import { RollbackWorkspace } from "./RollbackWorkspace.ts"; + +const threadId = ThreadId.make("thread-admission"); +const projectId = ProjectId.make("project-admission"); +const providerInstanceId = ProviderInstanceId.make("fake-absolute"); +const sessionIncarnationId = RuntimeSessionId.make("session-admission"); +const now = "2026-08-31T00:00:00.000Z"; +const baselineRef = checkpointRefForThreadTurn(threadId, 0); +const turnOneRef = checkpointRefForThreadTurn(threadId, 1); +const turnTwoRef = checkpointRefForThreadTurn(threadId, 2); + +type HarnessOptions = { + readonly mode?: "absolute" | "relative" | "unsupported"; + readonly queueCount?: number; + readonly workspaceMismatch?: boolean; + readonly activeLease?: boolean; + readonly missingAbsoluteMethod?: boolean; + readonly checkpoints?: ReadonlyArray; +}; + +const makeReadModel = (checkpoints: ReadonlyArray): OrchestrationReadModel => + ({ + snapshotSequence: 10, + projects: [ + { + id: projectId, + title: "Admission project", + workspaceRoot: "/workspace/exact", + defaultModelSelection: null, + defaultThreadEnvMode: null, + faviconPath: null, + scripts: [], + createdAt: now, + updatedAt: now, + deletedAt: null, + }, + ], + threads: [ + { + id: threadId, + projectId, + title: "Admission thread", + modelSelection: { instanceId: providerInstanceId, model: "fake-model" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + linkedPullRequest: null, + latestTurn: { + turnId: TurnId.make("turn-2"), + state: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + assistantMessageId: null, + }, + rollbackStatus: null, + createdAt: now, + updatedAt: now, + archivedAt: null, + settledOverride: null, + settledAt: null, + unsettledAt: null, + snoozedUntil: null, + snoozedAt: null, + pinnedAt: null, + pinOrderKey: null, + titleRegeneration: null, + continuedFromThreadId: null, + deletedAt: null, + messages: [], + proposedPlans: [], + activities: [], + checkpoints: checkpoints.map((checkpointTurnCount) => ({ + turnId: TurnId.make(`turn-${checkpointTurnCount}`), + checkpointTurnCount, + checkpointRef: checkpointTurnCount === 1 ? turnOneRef : turnTwoRef, + status: "ready" as const, + files: [], + assistantMessageId: null, + completedAt: now, + })), + session: { + threadId, + status: "idle", + providerName: "fake", + providerInstanceId, + activeTurnId: null, + startedAt: now, + updatedAt: now, + }, + }, + ], + updatedAt: now, + }) as unknown as OrchestrationReadModel; + +const makeHarness = (options: HarnessOptions = {}) => { + const mode = options.mode ?? "absolute"; + const checkpoints = options.checkpoints ?? [1, 2]; + const provider = { + getCapabilities: () => Effect.succeed({ conversationRollback: mode }), + hasAbsoluteConversationRollback: () => Effect.succeed(true), + captureConversationAnchor: () => Effect.succeed({ anchor: {}, digest: "source" }), + inspectConversationAnchor: () => Effect.succeed({ anchor: {}, digest: "source" }), + applyConversationAnchor: () => Effect.void, + getSessionInputQueue: () => + Effect.succeed({ + steeringCount: options.queueCount ?? 0, + followUpCount: 0, + mode: "steer", + steering: [], + followUps: [], + }), + listSessions: () => + Effect.succeed([ + { + threadId, + provider: "fake", + providerInstanceId, + sessionIncarnationId, + cwd: options.workspaceMismatch ? "/workspace/other" : "/workspace/exact", + status: "ready", + createdAt: now, + updatedAt: now, + }, + ]), + } as Record; + if (options.missingAbsoluteMethod === true) delete provider.applyConversationAnchor; + + const repository = { + getCheckpointAnchor: (input: { readonly checkpointTurnCount: number }) => + Effect.succeed( + Option.some({ + threadId, + checkpointTurnCount: input.checkpointTurnCount, + providerInstanceId, + sessionIncarnationId, + checkpointRef: input.checkpointTurnCount === 0 ? baselineRef : turnOneRef, + checkpointOid: input.checkpointTurnCount === 0 ? "0".repeat(40) : "1".repeat(40), + anchor: { leafId: `PRIVATE_TARGET_${input.checkpointTurnCount}` }, + anchorDigest: `target-${input.checkpointTurnCount}`, + capturedAt: now, + }), + ), + getActiveByThread: () => Effect.succeed(Option.none()), + findLeaseByWorkspace: () => + Effect.succeed( + options.activeLease === true + ? Option.some({ operationId: "other-operation", threadId, projectId }) + : Option.none(), + ), + } as unknown as RollbackSagaRepositoryShape; + + const workspace = { + resolveIdentity: (cwd: string) => + Effect.succeed({ + cwd, + workspaceKey: cwd === "/workspace/exact" ? "exact-key" : "other-key", + gitCommonDir: "/git/common", + }), + resolveCheckpoint: (input: { readonly checkpointRef: CheckpointRef }) => + Effect.succeed( + input.checkpointRef === turnTwoRef + ? { oid: "2".repeat(40), digest: "tree-2" } + : input.checkpointRef === turnOneRef + ? { oid: "1".repeat(40), digest: "tree-1" } + : { oid: "0".repeat(40), digest: "tree-0" }, + ), + }; + + const admission = makeRollbackAdmission.pipe( + Effect.provideService(ProviderService, provider as never), + Effect.provideService(RollbackSagaRepository, repository), + Effect.provideService(RollbackWorkspace, workspace as never), + Effect.provide(NodeServices.layer), + ); + return { admission, readModel: makeReadModel(checkpoints) }; +}; + +const prepare = Effect.fn(function* ( + options: HarnessOptions, + targetRevision: number, + expectedSourceRevision: number | "omit" = 2, +) { + const harness = makeHarness(options); + const admission = yield* harness.admission; + return yield* admission.prepare({ + command: { + type: "thread.checkpoint.revert", + commandId: "command-admission" as never, + threadId, + turnCount: targetRevision, + ...(expectedSourceRevision === "omit" ? {} : { expectedSourceRevision }), + createdAt: now, + }, + readModel: harness.readModel, + requestEventId: "request-event-admission", + }); +}); + +it.effect("admits turn 0 only from explicit immutable workspace and provider baselines", () => + Effect.gen(function* () { + const admitted = yield* prepare({}, 0); + assert.isTrue(Option.isSome(admitted)); + if (Option.isNone(admitted)) return; + assert.equal(admitted.value.sourceRevision, 2); + assert.equal(admitted.value.targetRevision, 0); + assert.equal(admitted.value.targetCheckpointRef, baselineRef); + assert.equal(admitted.value.targetCheckpointOid, "0".repeat(40)); + assert.deepEqual(admitted.value.desiredAnchor, { leafId: "PRIVATE_TARGET_0" }); + assert.equal(admitted.value.workspaceKey, "exact-key"); + }), +); + +it.effect( + "leaves every relative or unsupported production-style adapter on the fail-closed path", + () => + Effect.gen(function* () { + assert.isTrue(Option.isNone(yield* prepare({ mode: "relative" }, 1))); + assert.isTrue(Option.isNone(yield* prepare({ mode: "unsupported" }, 1))); + }), +); + +it.effect("requires the full absolute adapter contract and an empty provider queue", () => + Effect.gen(function* () { + const incomplete = yield* prepare({ missingAbsoluteMethod: true }, 1).pipe(Effect.result); + assert.equal(incomplete._tag, "Failure"); + const queued = yield* prepare({ queueCount: 1 }, 1).pipe(Effect.result); + assert.equal(queued._tag, "Failure"); + }), +); + +it.effect("rejects stale clients, partial checkpoint history, and mismatched workspaces", () => + Effect.gen(function* () { + assert.equal((yield* prepare({}, 1, "omit").pipe(Effect.result))._tag, "Failure"); + assert.equal((yield* prepare({}, 1, 1).pipe(Effect.result))._tag, "Failure"); + assert.equal((yield* prepare({ checkpoints: [2] }, 0).pipe(Effect.result))._tag, "Failure"); + assert.equal( + (yield* prepare({ workspaceMismatch: true }, 1).pipe(Effect.result))._tag, + "Failure", + ); + }), +); + +it.effect("rejects a second thread or client when the canonical workspace lease is active", () => + Effect.gen(function* () { + const result = yield* prepare({ activeLease: true }, 1).pipe(Effect.result); + assert.equal(result._tag, "Failure"); + }), +); diff --git a/apps/server/src/rollback/RollbackAdmission.ts b/apps/server/src/rollback/RollbackAdmission.ts new file mode 100644 index 000000000..6e4cc2d86 --- /dev/null +++ b/apps/server/src/rollback/RollbackAdmission.ts @@ -0,0 +1,287 @@ +import { + RuntimeSessionId, + type OrchestrationCommand, + type OrchestrationReadModel, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Layer from "effect/Layer"; +import { checkpointRefForThreadTurn } from "../checkpointing/Utils.ts"; +import { OrchestrationCommandInvariantError } from "../orchestration/Errors.ts"; +import { + RollbackSagaRepository, + type RollbackSagaState, +} from "../persistence/Services/RollbackSagas.ts"; +import { ProviderService } from "../provider/Services/ProviderService.ts"; +import { RollbackWorkspace } from "./RollbackWorkspace.ts"; + +function hasOpenInput(thread: OrchestrationReadModel["threads"][number]): boolean { + const open = new Set(); + for (const activity of thread.activities) { + const payload = + typeof activity.payload === "object" && activity.payload !== null + ? (activity.payload as Record) + : null; + const requestId = typeof payload?.requestId === "string" ? payload.requestId : null; + if (requestId === null) continue; + if ( + ["approval.requested", "user-input.requested", "interaction.requested"].includes( + activity.kind, + ) + ) { + open.add(requestId); + } else if ( + ["approval.resolved", "user-input.resolved", "interaction.resolved"].includes(activity.kind) + ) { + open.delete(requestId); + } + } + return open.size > 0; +} + +export interface RollbackAdmissionShape { + readonly prepare: (input: { + readonly command: Extract; + readonly readModel: OrchestrationReadModel; + readonly requestEventId: string; + }) => Effect.Effect, OrchestrationCommandInvariantError>; +} +export class RollbackAdmission extends Context.Service()( + "t3/rollback/RollbackAdmission", +) {} + +const invariant = (detail: string) => + new OrchestrationCommandInvariantError({ + commandType: "thread.checkpoint.revert", + detail, + }); + +export const make = Effect.gen(function* () { + const provider = yield* ProviderService; + const repository = yield* RollbackSagaRepository; + const workspace = yield* RollbackWorkspace; + const randomUUID = (yield* Crypto.Crypto).randomUUIDv4; + + const prepare: RollbackAdmissionShape["prepare"] = Effect.fn("RollbackAdmission.prepare")( + function* ({ command, readModel, requestEventId }) { + const thread = readModel.threads.find((candidate) => candidate.id === command.threadId); + if (!thread) return yield* invariant("Thread does not exist."); + const capabilities = yield* provider + .getCapabilities(thread.modelSelection.instanceId) + .pipe(Effect.option); + if (Option.isNone(capabilities) || capabilities.value.conversationRollback !== "absolute") { + return Option.none(); + } + if ( + provider.hasAbsoluteConversationRollback === undefined || + provider.captureConversationAnchor === undefined || + provider.inspectConversationAnchor === undefined || + provider.applyConversationAnchor === undefined || + !(yield* provider + .hasAbsoluteConversationRollback(command.threadId) + .pipe( + Effect.mapError(() => + invariant("The absolute provider rollback contract could not be verified."), + ), + )) + ) { + return yield* invariant("The absolute provider rollback contract is incomplete."); + } + if ( + thread.session === null || + !["idle", "ready"].includes(thread.session.status) || + thread.session.activeTurnId !== null || + thread.session.pendingTurnRequestId !== undefined || + thread.session.activeTurnRequestId !== undefined || + thread.session.failedTurnRequestId !== undefined || + thread.latestTurn?.state === "running" || + hasOpenInput(thread) + ) { + return yield* invariant( + "Rollback requires an exactly idle thread with no pending work or input.", + ); + } + + const queue = yield* provider + .getSessionInputQueue({ threadId: command.threadId }) + .pipe( + Effect.mapError(() => invariant("The provider input queue could not be proved empty.")), + ); + if (queue.steeringCount !== 0 || queue.followUpCount !== 0) { + return yield* invariant("Rollback requires an empty provider input queue."); + } + + const sessions = yield* provider + .listSessions() + .pipe(Effect.mapError(() => invariant("The exact provider session could not be listed."))); + const session = sessions.find((candidate) => candidate.threadId === command.threadId); + if ( + !session?.cwd || + !["idle", "ready"].includes(session.status) || + session.activeTurnId !== undefined || + session.providerInstanceId === undefined || + session.sessionIncarnationId === undefined + ) { + return yield* invariant("The exact idle provider session could not be resolved."); + } + const project = readModel.projects.find((candidate) => candidate.id === thread.projectId); + if (!project || project.deletedAt !== null) + return yield* invariant("Project does not exist."); + const configuredCwd = thread.worktreePath ?? project.workspaceRoot; + const sessionIdentity = yield* workspace + .resolveIdentity(session.cwd) + .pipe( + Effect.mapError(() => invariant("The provider workspace identity could not be proved.")), + ); + const configuredIdentity = yield* workspace + .resolveIdentity(configuredCwd) + .pipe( + Effect.mapError(() => + invariant("The configured workspace identity could not be proved."), + ), + ); + if (sessionIdentity.workspaceKey !== configuredIdentity.workspaceKey) { + return yield* invariant("The provider session is not bound to the exact thread workspace."); + } + + const sourceRevision = thread.checkpoints.reduce( + (max, checkpoint) => Math.max(max, checkpoint.checkpointTurnCount), + 0, + ); + const checkpointsByRevision = new Map( + thread.checkpoints.map((checkpoint) => [checkpoint.checkpointTurnCount, checkpoint]), + ); + for (let revision = 1; revision <= sourceRevision; revision += 1) { + if (!checkpointsByRevision.has(revision)) { + return yield* invariant("Rollback requires a complete contiguous checkpoint history."); + } + } + const checkpointTurnIds = new Set(thread.checkpoints.map((checkpoint) => checkpoint.turnId)); + if ( + thread.messages.some( + (message) => message.turnId !== null && !checkpointTurnIds.has(message.turnId), + ) + ) { + return yield* invariant("Rollback requires checkpoint history for every projected turn."); + } + if ( + command.expectedSourceRevision === undefined || + command.expectedSourceRevision !== sourceRevision + ) { + return yield* invariant("The rollback source revision is stale or absent."); + } + if (command.turnCount >= sourceRevision) { + return yield* invariant( + "The rollback target must be older than the exact source revision.", + ); + } + const sourceSummary = checkpointsByRevision.get(sourceRevision); + if (sourceSummary === undefined || sourceSummary.status !== "ready") { + return yield* invariant("The immutable source checkpoint is missing or corrupt."); + } + const targetSummary = + command.turnCount === 0 ? null : checkpointsByRevision.get(command.turnCount); + if (command.turnCount > 0 && (!targetSummary || targetSummary.status !== "ready")) { + return yield* invariant("The immutable target checkpoint is missing or corrupt."); + } + + const sourceCheckpointRef = checkpointRefForThreadTurn(command.threadId, sourceRevision); + const targetCheckpointRef = checkpointRefForThreadTurn(command.threadId, command.turnCount); + const sourceIdentity = yield* workspace + .resolveCheckpoint({ + cwd: sessionIdentity.cwd, + checkpointRef: sourceCheckpointRef, + }) + .pipe(Effect.mapError(() => invariant("The immutable source checkpoint is unavailable."))); + const targetIdentity = yield* workspace + .resolveCheckpoint({ + cwd: sessionIdentity.cwd, + checkpointRef: targetCheckpointRef, + }) + .pipe( + Effect.mapError(() => + invariant("The explicit immutable target checkpoint is unavailable."), + ), + ); + + const providerInstanceId = session.providerInstanceId; + const sessionIncarnationId = RuntimeSessionId.make(session.sessionIncarnationId); + const desired = yield* repository + .getCheckpointAnchor({ + threadId: command.threadId, + checkpointTurnCount: command.turnCount, + providerInstanceId, + sessionIncarnationId, + }) + .pipe( + Effect.mapError(() => invariant("The private target provider anchor is unavailable.")), + ); + if (Option.isNone(desired) || desired.value.checkpointOid !== targetIdentity.oid) { + return yield* invariant( + "The private target provider anchor does not match the immutable checkpoint.", + ); + } + if ( + Option.isSome( + yield* repository + .getActiveByThread(command.threadId) + .pipe(Effect.mapError(() => invariant("Rollback admission state could not be read."))), + ) + ) { + return yield* invariant("Another rollback operation is already active for this thread."); + } + if ( + Option.isSome( + yield* repository + .findLeaseByWorkspace(sessionIdentity.workspaceKey) + .pipe(Effect.mapError(() => invariant("The workspace lease could not be read."))), + ) + ) { + return yield* invariant("Another rollback operation owns this workspace."); + } + + const now = command.createdAt; + return Option.some({ + operationId: yield* randomUUID.pipe( + Effect.mapError(() => + invariant("A rollback operation identifier could not be generated."), + ), + ), + requestEventId, + threadId: command.threadId, + projectId: thread.projectId, + workspaceKey: sessionIdentity.workspaceKey, + workspaceCwd: sessionIdentity.cwd, + sourceRevision, + targetRevision: command.turnCount, + sourceCheckpointRef, + sourceCheckpointOid: sourceIdentity.oid, + targetCheckpointRef, + targetCheckpointOid: targetIdentity.oid, + targetCheckpointDigest: targetIdentity.digest, + providerInstanceId, + sessionIncarnationId, + phase: "source-anchor-capture-started", + attempt: 0, + lastErrorCode: null, + compensation: "none", + cleanup: "pending", + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: desired.value.anchor, + desiredAnchorDigest: desired.value.anchorDigest, + preimage: null, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: now, + updatedAt: now, + }); + }, + ); + return RollbackAdmission.of({ prepare }); +}); + +export const layer = Layer.effect(RollbackAdmission, make); diff --git a/apps/server/src/rollback/RollbackSagaRunner.test.ts b/apps/server/src/rollback/RollbackSagaRunner.test.ts new file mode 100644 index 000000000..0943809f6 --- /dev/null +++ b/apps/server/src/rollback/RollbackSagaRunner.test.ts @@ -0,0 +1,607 @@ +// @effect-diagnostics preferSchemaOverJson:off +import { + CheckpointRef, + ProjectId, + ProviderInstanceId, + RuntimeSessionId, + ThreadId, + type OrchestrationCommand, +} from "@t3tools/contracts"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Option from "effect/Option"; +import * as Stream from "effect/Stream"; + +import { CheckpointStore } from "../checkpointing/CheckpointStore.ts"; +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { + RuntimeReceiptBus, + type OrchestrationRuntimeReceipt, +} from "../orchestration/Services/RuntimeReceiptBus.ts"; +import { + RollbackSagaRepository, + type RollbackCheckpointAnchor, + type RollbackSagaRecord, + type RollbackSagaRepositoryShape, + type RollbackSagaState, +} from "../persistence/Services/RollbackSagas.ts"; +import { ProviderService } from "../provider/Services/ProviderService.ts"; +import { RollbackFaultInjector, make as makeRollbackSagaRunner } from "./RollbackSagaRunner.ts"; +import { RollbackWorkspace, type RollbackWorkspacePreimage } from "./RollbackWorkspace.ts"; + +const threadId = ThreadId.make("thread-runner"); +const projectId = ProjectId.make("project-runner"); +const providerInstanceId = ProviderInstanceId.make("fake-absolute"); +const sessionIncarnationId = RuntimeSessionId.make("fake-session-incarnation"); +const now = "2026-08-31T00:00:00.000Z"; +const privateTargetCanary = "PRIVATE_TARGET_LEAF_CANARY"; +const privateSourceCanary = "PRIVATE_SOURCE_LEAF_CANARY"; +const privatePreimageCanary = "/private/preimage/canary"; + +type ProviderMode = "success" | "unknown-target" | "stayed-source" | "wrong-target"; + +const makeState = (operationId: string): RollbackSagaState => ({ + operationId, + requestEventId: `event-${operationId}`, + threadId, + projectId, + workspaceKey: `workspace-${operationId}`, + workspaceCwd: "/workspace/fake", + sourceRevision: 2, + targetRevision: 1, + sourceCheckpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-runner/turn/2"), + sourceCheckpointOid: "a".repeat(40), + targetCheckpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-runner/turn/1"), + targetCheckpointOid: "b".repeat(40), + targetCheckpointDigest: "target-tree-digest", + providerInstanceId, + sessionIncarnationId, + phase: "source-anchor-capture-started", + attempt: 0, + lastErrorCode: null, + compensation: "none", + cleanup: "pending", + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: { leafId: privateTargetCanary }, + desiredAnchorDigest: "provider-target", + preimage: null, + workspaceReceiptDigest: null, + providerReceiptDigest: null, + projectionCommitSequence: null, + createdAt: now, + updatedAt: now, +}); + +const makeEnvironment = ( + operationId: string, + providerMode: ProviderMode = "success", + cleanupFailures = 0, + projectionCommitFails = false, +) => { + let record: RollbackSagaRecord = { + operationId, + requestEventId: `event-${operationId}`, + threadId, + projectId, + workspaceKey: `workspace-${operationId}`, + phase: "source-anchor-capture-started", + terminal: false, + ownerId: null, + version: 0, + state: makeState(operationId), + createdAt: now, + updatedAt: now, + }; + let lease = true; + let providerDigest = "provider-source"; + let workspaceDigest = "workspace-source"; + let preimageCleaned = false; + let anchorsDeleted = false; + let staleRefsDeleted = false; + let projectionCommitted = false; + let projectionCommits = 0; + const commands: OrchestrationCommand[] = []; + const runtimeReceipts: OrchestrationRuntimeReceipt[] = []; + + const preimage: RollbackWorkspacePreimage = { + backupPath: privatePreimageCanary, + digest: "workspace-source", + indexPath: "/private/index/canary", + indexExisted: true, + headSymbolic: "refs/heads/main", + headOid: "f".repeat(40), + ownedRefs: [], + paths: ["tracked.txt"], + entryCount: 3, + totalBytes: 42, + }; + + const repository: RollbackSagaRepositoryShape = { + admit: () => Effect.void, + get: (id) => Effect.succeed(id === operationId ? Option.some(record) : Option.none()), + getByRequestEvent: (eventId) => + Effect.succeed(eventId === record.requestEventId ? Option.some(record) : Option.none()), + getActiveByThread: (id) => + Effect.succeed(id === threadId && !record.terminal ? Option.some(record) : Option.none()), + listNonterminal: () => Effect.succeed(record.terminal ? [] : [record]), + listNonterminalForFence: () => Effect.succeed(record.terminal ? [] : [record]), + clearOwnersForStartup: () => + Effect.sync(() => { + if (!record.terminal) record = { ...record, ownerId: null }; + }), + claim: (id, ownerId) => + Effect.sync(() => { + if ( + id !== operationId || + record.terminal || + (record.ownerId !== null && record.ownerId !== ownerId) + ) { + return Option.none(); + } + record = { ...record, ownerId }; + return Option.some(record); + }), + updateOwned: (input) => + Effect.sync(() => { + if ( + record.terminal || + record.operationId !== input.operationId || + record.ownerId !== input.ownerId || + record.version !== input.expectedVersion + ) + return Option.none(); + record = { + ...record, + phase: input.state.phase, + terminal: input.terminal === true, + version: record.version + 1, + state: input.state, + updatedAt: input.state.updatedAt, + }; + return Option.some(record); + }), + releaseOwnerOwned: (id, ownerId) => + Effect.sync(() => { + if (record.operationId === id && record.ownerId === ownerId && !record.terminal) { + record = { ...record, ownerId: null }; + } + }), + releaseLeaseOwned: (input) => + Effect.sync(() => { + if ( + record.terminal || + record.operationId !== input.operationId || + record.ownerId !== input.ownerId || + record.version !== input.expectedVersion + ) + return Option.none(); + record = { + ...record, + phase: input.state.phase, + terminal: true, + ownerId: null, + version: record.version + 1, + state: input.state, + updatedAt: input.state.updatedAt, + }; + lease = false; + return Option.some(record); + }), + findLeaseByWorkspace: () => + Effect.succeed(lease ? Option.some({ operationId, threadId, projectId }) : Option.none()), + putCheckpointAnchor: () => Effect.void, + getCheckpointAnchor: () => Effect.succeed(Option.none()), + deleteCheckpointAnchorsAfter: () => + Effect.sync(() => { + anchorsDeleted = true; + }), + }; + + const provider = { + captureConversationAnchor: () => + Effect.succeed({ + anchor: { leafId: privateSourceCanary }, + digest: providerDigest, + }), + inspectConversationAnchor: () => + Effect.sync(() => ({ + anchor: { leafId: providerDigest }, + digest: providerDigest, + })), + applyConversationAnchor: (input: { readonly anchor: unknown }) => + Effect.suspend(() => { + const isTarget = + (input.anchor as { readonly leafId?: string }).leafId === privateTargetCanary; + if (!isTarget) { + providerDigest = "provider-source"; + return Effect.void; + } + switch (providerMode) { + case "success": + providerDigest = "provider-target"; + return Effect.void; + case "unknown-target": + providerDigest = "provider-target"; + return Effect.fail("provider-timeout"); + case "stayed-source": + return Effect.fail("provider-failed"); + case "wrong-target": + providerDigest = "provider-wrong"; + return Effect.void; + } + }), + }; + + const workspace = { + capturePreimage: () => Effect.succeed(preimage), + applyCheckpoint: () => + Effect.sync(() => { + workspaceDigest = "workspace-target"; + return { + digest: workspaceDigest, + headSymbolic: "refs/heads/main", + headOid: "f".repeat(40), + }; + }), + restorePreimage: () => + Effect.sync(() => { + workspaceDigest = "workspace-source"; + return { + digest: workspaceDigest, + headSymbolic: "refs/heads/main", + headOid: "f".repeat(40), + }; + }), + inspect: () => + Effect.succeed({ + digest: workspaceDigest, + treeDigest: workspaceDigest, + headSymbolic: "refs/heads/main", + headOid: "f".repeat(40), + }), + inspectCheckpoint: () => + Effect.succeed({ + digest: workspaceDigest, + treeDigest: workspaceDigest, + headSymbolic: "refs/heads/main", + headOid: "f".repeat(40), + }), + cleanupPreimage: () => + Effect.suspend(() => { + if (cleanupFailures > 0) { + cleanupFailures -= 1; + return Effect.fail("cleanup-failed"); + } + preimageCleaned = true; + return Effect.void; + }), + }; + + const engine = { + dispatch: (command: OrchestrationCommand) => + Effect.suspend(() => { + commands.push(command); + if (command.type === "thread.revert.complete" && projectionCommitFails) { + return Effect.fail("projection-cas-failed"); + } + if (command.type === "thread.revert.complete" && !projectionCommitted) { + projectionCommitted = true; + projectionCommits += 1; + } + return Effect.succeed({ sequence: 100, eventCount: 1 }); + }), + readEvents: () => Stream.empty, + streamDomainEvents: Stream.empty, + latestSequence: Effect.succeed(100), + }; + + const checkpointStore = { + deleteCheckpointRefs: () => + Effect.sync(() => { + staleRefsDeleted = true; + }), + }; + const receipts = { + publish: (receipt: OrchestrationRuntimeReceipt) => + Effect.sync(() => { + runtimeReceipts.push(receipt); + }), + streamEventsForTest: Stream.empty, + }; + + const makeRunner = (faultLabel: string | null = null) => { + let armed = true; + const fault = (label: string) => { + if (armed && label === faultLabel) { + armed = false; + return Effect.interrupt; + } + return Effect.void; + }; + return makeRollbackSagaRunner.pipe( + Effect.provideService(RollbackSagaRepository, repository), + Effect.provideService(RollbackWorkspace, workspace as never), + Effect.provideService(ProviderService, provider as never), + Effect.provideService(OrchestrationEngineService, engine as never), + Effect.provideService(CheckpointStore, checkpointStore as never), + Effect.provideService(RuntimeReceiptBus, receipts), + Effect.provideService(RollbackFaultInjector, fault), + Effect.provide(NodeServices.layer), + ); + }; + + return { + repository, + makeRunner, + snapshot: () => ({ + record, + lease, + providerDigest, + workspaceDigest, + preimageCleaned, + anchorsDeleted, + staleRefsDeleted, + projectionCommitted, + projectionCommits, + commands, + runtimeReceipts, + }), + }; +}; + +const runInterrupted = Effect.fn(function* ( + runner: { readonly run: (operationId: string, recovering: boolean) => Effect.Effect }, + operationId: string, +) { + const fiber = yield* runner + .run(operationId, false) + .pipe(Effect.forkChild({ startImmediately: true })); + return yield* Fiber.await(fiber); +}); + +it.effect("commits last, clears private state, and never publishes private canaries", () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-success"); + const runner = yield* environment.makeRunner(); + yield* runner.run("operation-success", false); + const snapshot = environment.snapshot(); + + assert.equal(snapshot.record.state.phase, "complete"); + assert.isTrue(snapshot.record.terminal); + assert.isFalse(snapshot.lease); + assert.equal(snapshot.workspaceDigest, "workspace-target"); + assert.equal(snapshot.providerDigest, "provider-target"); + assert.equal(snapshot.projectionCommits, 1); + assert.isTrue(snapshot.preimageCleaned); + assert.isTrue(snapshot.anchorsDeleted); + assert.isTrue(snapshot.staleRefsDeleted); + assert.equal(snapshot.record.state.sourceAnchor, null); + assert.equal(snapshot.record.state.desiredAnchor, null); + assert.equal(snapshot.record.state.preimage, null); + + const publicJson = JSON.stringify({ + commands: snapshot.commands, + receipts: snapshot.runtimeReceipts, + }); + assert.notInclude(publicJson, privateTargetCanary); + assert.notInclude(publicJson, privateSourceCanary); + assert.notInclude(publicJson, privatePreimageCanary); + const completeIndex = snapshot.commands.findIndex( + (command) => command.type === "thread.revert.complete", + ); + assert.isAtLeast(completeIndex, 0); + assert.isTrue( + snapshot.runtimeReceipts.some( + (receipt) => receipt.type === "rollback.saga.phase" && receipt.phase === "complete", + ), + ); + }), +); + +it.effect("reconciles an unknown provider result by inspecting the exact target", () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-unknown-target", "unknown-target"); + const runner = yield* environment.makeRunner(); + yield* runner.run("operation-unknown-target", false); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "complete"); + assert.equal(snapshot.providerDigest, "provider-target"); + assert.equal(snapshot.projectionCommits, 1); + }), +); + +it.effect("compensates workspace and provider when the provider stays at source", () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-compensate", "stayed-source"); + const runner = yield* environment.makeRunner(); + yield* runner.run("operation-compensate", false); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "compensated"); + assert.isTrue(snapshot.record.terminal); + assert.equal(snapshot.workspaceDigest, "workspace-source"); + assert.equal(snapshot.providerDigest, "provider-source"); + assert.equal(snapshot.projectionCommits, 0); + assert.isFalse(snapshot.lease); + }), +); + +it.effect( + "fails closed in manual recovery when inspection finds an unrelated provider anchor", + () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-wrong-target", "wrong-target"); + const runner = yield* environment.makeRunner(); + yield* runner.run("operation-wrong-target", false); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "manual-recovery"); + assert.isFalse(snapshot.record.terminal); + assert.isTrue(snapshot.lease); + assert.equal(snapshot.workspaceDigest, "workspace-target"); + assert.equal(snapshot.providerDigest, "provider-wrong"); + assert.equal(snapshot.projectionCommits, 0); + assert.isTrue( + snapshot.commands.some( + (command) => + command.type === "thread.rollback.status.set" && command.status === "manual-recovery", + ), + ); + + const restarted = yield* environment.makeRunner(); + yield* restarted.run("operation-wrong-target", true); + const statusCommands = environment + .snapshot() + .commands.filter((command) => command.type === "thread.rollback.status.set"); + assert.equal(statusCommands.at(-1)?.status, "manual-recovery"); + }), +); + +it.effect("releases only the worker owner and retries post-commit cleanup idempotently", () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-cleanup-retry", "success", 1); + const firstRunner = yield* environment.makeRunner(); + yield* firstRunner.run("operation-cleanup-retry", false); + const pending = environment.snapshot(); + assert.equal(pending.record.state.phase, "cleanup-started"); + assert.isFalse(pending.record.terminal); + assert.equal(pending.record.ownerId, null); + assert.isTrue(pending.lease); + assert.equal(pending.projectionCommits, 1); + + const retryRunner = yield* environment.makeRunner(); + yield* retryRunner.run("operation-cleanup-retry", true); + const complete = environment.snapshot(); + assert.equal(complete.record.state.phase, "complete"); + assert.isTrue(complete.record.terminal); + assert.isFalse(complete.lease); + assert.equal(complete.projectionCommits, 1); + assert.isTrue(complete.preimageCleaned); + }), +); + +it.effect("compensates a proved provider and workspace target when projection CAS fails", () => + Effect.gen(function* () { + const environment = makeEnvironment("operation-projection-cas", "success", 0, true); + const runner = yield* environment.makeRunner(); + yield* runner.run("operation-projection-cas", false); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "compensated"); + assert.isTrue(snapshot.record.terminal); + assert.equal(snapshot.workspaceDigest, "workspace-source"); + assert.equal(snapshot.providerDigest, "provider-source"); + assert.equal(snapshot.projectionCommits, 0); + assert.isFalse(snapshot.lease); + }), +); + +const compensationFaultLabels = [ + "persisted:compensation-workspace-started", + "side-effect:workspace-compensated", + "persisted:compensation-workspace-complete", + "persisted:compensation-provider-started", + "side-effect:provider-compensated", + "persisted:compensated", +] as const; + +for (const faultLabel of compensationFaultLabels) { + it.effect(`reconciles compensation after a crash at ${faultLabel}`, () => + Effect.gen(function* () { + const operationId = `operation-compensation-fault-${faultLabel}`; + const environment = makeEnvironment(operationId, "success", 0, true); + const interruptedRunner = yield* environment.makeRunner(faultLabel); + const exit = yield* runInterrupted(interruptedRunner, operationId); + assert.equal(exit._tag, "Failure"); + + yield* environment.repository.clearOwnersForStartup(); + const restartedRunner = yield* environment.makeRunner(); + yield* restartedRunner.run(operationId, true); + const snapshot = environment.snapshot(); + assert.equal( + snapshot.record.state.phase, + "compensated", + JSON.stringify({ + faultLabel, + error: snapshot.record.state.lastErrorCode, + compensation: snapshot.record.state.compensation, + workspace: snapshot.workspaceDigest, + provider: snapshot.providerDigest, + }), + ); + assert.isTrue(snapshot.record.terminal); + assert.equal(snapshot.workspaceDigest, "workspace-source"); + assert.equal(snapshot.providerDigest, "provider-source"); + assert.equal(snapshot.projectionCommits, 0); + assert.isFalse(snapshot.lease); + }), + ); +} + +it.effect("restores durable manual recovery status after a crash before status projection", () => + Effect.gen(function* () { + const operationId = "operation-manual-status-crash"; + const environment = makeEnvironment(operationId, "wrong-target"); + const interruptedRunner = yield* environment.makeRunner("persisted:manual-recovery"); + const exit = yield* runInterrupted(interruptedRunner, operationId); + assert.equal(exit._tag, "Failure"); + + yield* environment.repository.clearOwnersForStartup(); + const restartedRunner = yield* environment.makeRunner(); + yield* restartedRunner.run(operationId, true); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "manual-recovery"); + assert.isFalse(snapshot.record.terminal); + assert.isTrue(snapshot.lease); + const statuses = snapshot.commands.filter( + (command) => command.type === "thread.rollback.status.set", + ); + assert.equal(statuses.at(-1)?.status, "manual-recovery"); + }), +); + +const restartFaultLabels = [ + "side-effect:source-anchor-captured", + "persisted:source-anchor-captured", + "persisted:preimage-capture-started", + "side-effect:preimage-captured", + "persisted:preimage-captured", + "persisted:workspace-apply-started", + "side-effect:workspace-target-applied", + "persisted:workspace-applied", + "persisted:provider-apply-started", + "side-effect:provider-target-applied", + "persisted:provider-applied", + "persisted:projection-commit-started", + "side-effect:projection-committed", + "persisted:projection-committed", + "persisted:cleanup-started", + "side-effect:cleanup", + "persisted:complete", +] as const; + +for (const faultLabel of restartFaultLabels) { + it.effect(`reconciles deterministically after a crash at ${faultLabel}`, () => + Effect.gen(function* () { + const operationId = `operation-fault-${faultLabel}`; + const environment = makeEnvironment(operationId); + const interruptedRunner = yield* environment.makeRunner(faultLabel); + const exit = yield* runInterrupted(interruptedRunner, operationId); + assert.equal(exit._tag, "Failure"); + + yield* environment.repository.clearOwnersForStartup(); + const restartedRunner = yield* environment.makeRunner(); + yield* restartedRunner.run(operationId, true); + const snapshot = environment.snapshot(); + assert.equal(snapshot.record.state.phase, "complete"); + assert.isTrue(snapshot.record.terminal); + assert.isFalse(snapshot.lease); + assert.equal(snapshot.workspaceDigest, "workspace-target"); + assert.equal(snapshot.providerDigest, "provider-target"); + assert.equal(snapshot.projectionCommits, 1); + assert.isTrue(snapshot.preimageCleaned); + assert.isTrue(snapshot.anchorsDeleted); + assert.isTrue(snapshot.staleRefsDeleted); + }), + ); +} diff --git a/apps/server/src/rollback/RollbackSagaRunner.ts b/apps/server/src/rollback/RollbackSagaRunner.ts new file mode 100644 index 000000000..f4efecaf9 --- /dev/null +++ b/apps/server/src/rollback/RollbackSagaRunner.ts @@ -0,0 +1,488 @@ +import { CommandId, type CheckpointRef } from "@t3tools/contracts"; +import * as Cause from "effect/Cause"; +import * as Context from "effect/Context"; +import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import { checkpointRefForThreadTurn } from "../checkpointing/Utils.ts"; +import { CheckpointStore } from "../checkpointing/CheckpointStore.ts"; +import { OrchestrationEngineService } from "../orchestration/Services/OrchestrationEngine.ts"; +import { RuntimeReceiptBus } from "../orchestration/Services/RuntimeReceiptBus.ts"; +import { + RollbackSagaRepository, + type RollbackSagaRecord, + type RollbackSagaState, +} from "../persistence/Services/RollbackSagas.ts"; +import { ProviderService } from "../provider/Services/ProviderService.ts"; +import { RollbackWorkspace, type RollbackWorkspacePreimage } from "./RollbackWorkspace.ts"; + +export type RollbackFaultHook = (label: string, operationId: string) => Effect.Effect; +export const RollbackFaultInjector = Context.Reference( + "t3/rollback/RollbackFaultInjector", + { defaultValue: () => () => Effect.void }, +); + +export interface RollbackSagaRunnerShape { + readonly run: (operationId: string, recovering: boolean) => Effect.Effect; +} +export class RollbackSagaRunner extends Context.Service< + RollbackSagaRunner, + RollbackSagaRunnerShape +>()("t3/rollback/RollbackSagaRunner") {} + +const MAX_PROVIDER_TARGET_ATTEMPTS = 3; +const privatePreimage = (state: RollbackSagaState) => + state.preimage as RollbackWorkspacePreimage | null; + +export const make = Effect.gen(function* () { + const repository = yield* RollbackSagaRepository; + const workspace = yield* RollbackWorkspace; + const provider = yield* ProviderService; + const engine = yield* OrchestrationEngineService; + const checkpointStore = yield* CheckpointStore; + const receipts = yield* RuntimeReceiptBus; + const fault = yield* RollbackFaultInjector; + const captureConversationAnchor = provider.captureConversationAnchor; + const inspectConversationAnchor = provider.inspectConversationAnchor; + const applyConversationAnchor = provider.applyConversationAnchor; + const ownerId = yield* (yield* Crypto.Crypto).randomUUIDv4; + const nowIso = Effect.map(DateTime.now, DateTime.formatIso); + + const publishPhase = (state: RollbackSagaState) => + receipts.publish({ + type: "rollback.saga.phase", + operationId: state.operationId, + phase: state.phase, + createdAt: state.updatedAt, + }); + const after = (label: string, operationId: string) => fault(label, operationId); + const statusCommand = Effect.fn("RollbackSagaRunner.statusCommand")(function* ( + state: RollbackSagaState, + status: "pending" | "recovering" | "manual-recovery" | null, + ) { + const createdAt = yield* nowIso; + yield* engine.dispatch({ + type: "thread.rollback.status.set", + commandId: CommandId.make(`server:rollback-status:${state.operationId}:${status ?? "clear"}`), + threadId: state.threadId, + status, + createdAt, + }); + }); + + const update = Effect.fn("RollbackSagaRunner.update")(function* ( + record: RollbackSagaRecord, + patch: Partial & Pick, + terminal = false, + ) { + const state = { + ...record.state, + ...patch, + updatedAt: yield* nowIso, + } satisfies RollbackSagaState; + const updated = yield* repository.updateOwned({ + operationId: record.operationId, + ownerId, + expectedVersion: record.version, + state, + terminal, + }); + if (Option.isNone(updated)) return Option.none(); + yield* publishPhase(updated.value.state); + yield* after(`persisted:${state.phase}`, state.operationId); + return updated; + }); + + const manual = Effect.fn("RollbackSagaRunner.manual")(function* ( + record: RollbackSagaRecord, + code: string, + ) { + const next = yield* update(record, { + phase: "manual-recovery", + compensation: "manual", + lastErrorCode: code, + }); + if (Option.isSome(next)) + yield* statusCommand(next.value.state, "manual-recovery").pipe(Effect.ignore); + }); + + const releaseTerminal = Effect.fn("RollbackSagaRunner.releaseTerminal")(function* ( + record: RollbackSagaRecord, + state: RollbackSagaState, + ) { + const released = yield* repository.releaseLeaseOwned({ + operationId: record.operationId, + ownerId, + expectedVersion: record.version, + state: { ...state, updatedAt: yield* nowIso }, + }); + if (Option.isSome(released)) { + yield* publishPhase(released.value.state); + yield* after(`persisted:${released.value.state.phase}`, released.value.operationId); + } + }); + + const compensate = Effect.fn("RollbackSagaRunner.compensate")(function* ( + initial: RollbackSagaRecord, + code: string, + ) { + let record = initial; + let workspaceProved = privatePreimage(record.state) === null; + let providerProved = record.state.sourceAnchorDigest === null; + + const workspaceStarted = yield* update(record, { + phase: "compensation-workspace-started", + compensation: "workspace", + lastErrorCode: code, + }); + if (Option.isNone(workspaceStarted)) return; + record = workspaceStarted.value; + const preimage = privatePreimage(record.state); + if (preimage !== null) { + workspaceProved = yield* workspace + .restorePreimage({ + cwd: record.state.workspaceCwd, + preimage, + }) + .pipe( + Effect.tap(() => after("side-effect:workspace-compensated", record.operationId)), + Effect.match({ + onFailure: () => false, + onSuccess: (receipt) => receipt.digest === preimage.digest, + }), + ); + } + + const workspaceComplete = yield* update(record, { + phase: "compensation-workspace-complete", + compensation: "provider", + lastErrorCode: workspaceProved ? code : "workspace-compensation-unproved", + }); + if (Option.isNone(workspaceComplete)) return; + record = workspaceComplete.value; + + const providerStarted = yield* update(record, { + phase: "compensation-provider-started", + compensation: "provider", + }); + if (Option.isNone(providerStarted)) return; + record = providerStarted.value; + if (record.state.sourceAnchor !== null && record.state.sourceAnchorDigest !== null) { + providerProved = yield* inspectConversationAnchor!(record.state.threadId).pipe( + Effect.flatMap((inspected) => { + if (inspected.digest === record.state.sourceAnchorDigest) return Effect.succeed(true); + return applyConversationAnchor!({ + threadId: record.state.threadId, + anchor: record.state.sourceAnchor, + }).pipe( + Effect.tap(() => after("side-effect:provider-compensated", record.operationId)), + Effect.andThen(inspectConversationAnchor!(record.state.threadId)), + Effect.map((post) => post.digest === record.state.sourceAnchorDigest), + ); + }), + Effect.orElseSucceed(() => false), + ); + } + + if (!workspaceProved || !providerProved) { + yield* manual( + record, + !workspaceProved ? "workspace-compensation-unproved" : "provider-compensation-unproved", + ); + return; + } + const preimageForCleanup = privatePreimage(record.state); + if (preimageForCleanup !== null) { + const cleaned = yield* workspace.cleanupPreimage(preimageForCleanup).pipe(Effect.result); + if (cleaned._tag === "Failure") { + yield* manual(record, "cleanup-failed-after-compensation"); + return; + } + } + const terminalState: RollbackSagaState = { + ...record.state, + phase: "compensated", + compensation: "proved", + cleanup: "complete", + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: null, + desiredAnchorDigest: null, + preimage: null, + updatedAt: yield* nowIso, + }; + yield* statusCommand(terminalState, null).pipe(Effect.ignore); + yield* releaseTerminal(record, terminalState); + }); + + const step = Effect.fn("RollbackSagaRunner.step")(function* (initial: RollbackSagaRecord) { + let record = initial; + while (true) { + const state = record.state; + switch (state.phase) { + case "source-anchor-capture-started": { + const source = yield* captureConversationAnchor!(state.threadId).pipe(Effect.result); + yield* after("side-effect:source-anchor-captured", state.operationId); + if (source._tag === "Failure") + return yield* compensate(record, "source-anchor-capture-failed"); + const next = yield* update(record, { + phase: "source-anchor-captured", + sourceAnchor: source.success.anchor, + sourceAnchorDigest: source.success.digest, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "source-anchor-captured": { + const next = yield* update(record, { phase: "preimage-capture-started" }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "preimage-capture-started": { + const captured = yield* workspace + .capturePreimage({ + operationId: state.operationId, + cwd: state.workspaceCwd, + targetCheckpointOid: state.targetCheckpointOid, + }) + .pipe(Effect.result); + yield* after("side-effect:preimage-captured", state.operationId); + if (captured._tag === "Failure") + return yield* compensate(record, "preimage-capture-failed"); + const next = yield* update(record, { + phase: "preimage-captured", + preimage: captured.success, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "preimage-captured": { + const next = yield* update(record, { phase: "workspace-apply-started" }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "workspace-apply-started": { + const applied = yield* workspace + .applyCheckpoint({ + cwd: state.workspaceCwd, + checkpointOid: state.targetCheckpointOid, + }) + .pipe(Effect.result); + yield* after("side-effect:workspace-target-applied", state.operationId); + if (applied._tag === "Failure") + return yield* compensate(record, "workspace-target-failed"); + const next = yield* update(record, { + phase: "workspace-applied", + workspaceReceiptDigest: applied.success.digest, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "workspace-applied": { + const next = yield* update(record, { phase: "provider-apply-started" }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "provider-apply-started": { + if ( + state.sourceAnchorDigest === null || + state.desiredAnchorDigest === null || + state.desiredAnchor === null + ) { + return yield* manual(record, "provider-anchor-receipt-missing"); + } + const before = yield* inspectConversationAnchor!(state.threadId).pipe(Effect.result); + if (before._tag === "Success" && before.success.digest === state.desiredAnchorDigest) { + const next = yield* update(record, { + phase: "provider-applied", + providerReceiptDigest: before.success.digest, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + if (before._tag === "Success" && before.success.digest !== state.sourceAnchorDigest) { + return yield* manual(record, "provider-anchor-neither-source-nor-target"); + } + const applied = yield* applyConversationAnchor!({ + threadId: state.threadId, + anchor: state.desiredAnchor, + }).pipe(Effect.result); + yield* after("side-effect:provider-target-applied", state.operationId); + const inspected = yield* inspectConversationAnchor!(state.threadId).pipe(Effect.result); + if ( + inspected._tag === "Success" && + inspected.success.digest === state.desiredAnchorDigest + ) { + const next = yield* update(record, { + phase: "provider-applied", + providerReceiptDigest: inspected.success.digest, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + if ( + inspected._tag === "Success" && + inspected.success.digest === state.sourceAnchorDigest + ) { + if (state.attempt + 1 < MAX_PROVIDER_TARGET_ATTEMPTS) { + const next = yield* update(record, { + phase: "provider-apply-started", + attempt: state.attempt + 1, + lastErrorCode: + applied._tag === "Failure" + ? "provider-target-unknown-source" + : "provider-target-stayed-source", + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + return yield* compensate(record, "provider-target-retry-exhausted"); + } + return yield* manual(record, "provider-target-outcome-unknown"); + } + case "provider-applied": { + const workspaceReceipt = yield* workspace + .inspectCheckpoint({ + cwd: state.workspaceCwd, + checkpointOid: state.targetCheckpointOid, + }) + .pipe(Effect.result); + const providerReceipt = yield* inspectConversationAnchor!(state.threadId).pipe( + Effect.result, + ); + if ( + workspaceReceipt._tag === "Failure" || + workspaceReceipt.success.digest !== state.workspaceReceiptDigest || + providerReceipt._tag === "Failure" || + providerReceipt.success.digest !== state.desiredAnchorDigest + ) + return yield* manual(record, "precommit-postcondition-lost"); + const next = yield* update(record, { phase: "projection-commit-started" }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "projection-commit-started": { + const committed = yield* engine + .dispatch({ + type: "thread.revert.complete", + commandId: CommandId.make(`server:rollback-complete:${state.operationId}`), + threadId: state.threadId, + operationId: state.operationId, + sourceRevision: state.sourceRevision, + targetRevision: state.targetRevision, + turnCount: state.targetRevision, + createdAt: yield* nowIso, + }) + .pipe(Effect.result); + yield* after("side-effect:projection-committed", state.operationId); + if (committed._tag === "Failure") { + return yield* compensate(record, "projection-commit-cas-failed"); + } + const next = yield* update(record, { + phase: "projection-committed", + projectionCommitSequence: committed.success.sequence, + }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "projection-committed": { + const next = yield* update(record, { phase: "cleanup-started", cleanup: "running" }); + if (Option.isNone(next)) return; + record = next.value; + continue; + } + case "cleanup-started": { + const staleRefs: CheckpointRef[] = []; + for ( + let revision = state.targetRevision + 1; + revision <= state.sourceRevision; + revision += 1 + ) { + staleRefs.push(checkpointRefForThreadTurn(state.threadId, revision)); + } + const cleanup = yield* Effect.gen(function* () { + if (staleRefs.length > 0) + yield* checkpointStore.deleteCheckpointRefs({ + cwd: state.workspaceCwd, + checkpointRefs: staleRefs, + }); + yield* repository.deleteCheckpointAnchorsAfter({ + threadId: state.threadId, + checkpointTurnCount: state.targetRevision, + }); + const preimage = privatePreimage(state); + if (preimage !== null) yield* workspace.cleanupPreimage(preimage); + }).pipe(Effect.result); + yield* after("side-effect:cleanup", state.operationId); + if (cleanup._tag === "Failure") return; + const terminalState: RollbackSagaState = { + ...state, + phase: "complete", + cleanup: "complete", + sourceAnchor: null, + sourceAnchorDigest: null, + desiredAnchor: null, + desiredAnchorDigest: null, + preimage: null, + updatedAt: yield* nowIso, + }; + yield* releaseTerminal(record, terminalState); + return; + } + case "compensation-workspace-started": + case "compensation-workspace-complete": + case "compensation-provider-started": + return yield* compensate(record, state.lastErrorCode ?? "reconcile-compensation"); + case "manual-recovery": + case "compensated": + case "complete": + return; + } + } + }); + + const run: RollbackSagaRunnerShape["run"] = Effect.fn("RollbackSagaRunner.run")( + function* (operationId, recovering) { + const claimed = yield* repository.claim(operationId, ownerId).pipe(Effect.result); + if (claimed._tag === "Failure" || Option.isNone(claimed.success)) return; + const record = claimed.success.value; + yield* statusCommand( + record.state, + record.state.phase === "manual-recovery" + ? "manual-recovery" + : recovering + ? "recovering" + : "pending", + ).pipe(Effect.ignore); + yield* step(record).pipe( + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.interrupt; + return repository.get(operationId).pipe( + Effect.flatMap((latest) => + Option.isSome(latest) ? manual(latest.value, "unexpected-saga-failure") : Effect.void, + ), + Effect.ignore, + ); + }), + ); + yield* repository.releaseOwnerOwned(operationId, ownerId).pipe(Effect.ignore); + }, + ); + + return RollbackSagaRunner.of({ run }); +}); + +export const layer = Layer.effect(RollbackSagaRunner, make); diff --git a/apps/server/src/rollback/RollbackWorkspace.test.ts b/apps/server/src/rollback/RollbackWorkspace.test.ts new file mode 100644 index 000000000..9aa0b9c2e --- /dev/null +++ b/apps/server/src/rollback/RollbackWorkspace.test.ts @@ -0,0 +1,287 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeOS from "node:os"; +import * as NodeUtil from "node:util"; + +import * as ServerConfig from "../config.ts"; +import { RollbackWorkspace, layer as RollbackWorkspaceLive } from "./RollbackWorkspace.ts"; + +const execFileAsync = NodeUtil.promisify(NodeChildProcess.execFile); +const run = async (cwd: string, args: ReadonlyArray) => + (await execFileAsync("git", [...args], { cwd, encoding: "utf8", timeout: 30_000 })).stdout.trim(); + +const layer = it.layer( + RollbackWorkspaceLive.pipe( + Layer.provide( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-rollback-workspace-test-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ), + ), +); + +layer("RollbackWorkspace", (it) => { + it.effect("applies an immutable target and restores the complete mutable Git pre-image", () => + Effect.acquireUseRelease( + Effect.tryPromise(() => NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-rollback-git-"))), + (cwd) => + Effect.gen(function* () { + const workspace = yield* RollbackWorkspace; + + yield* Effect.promise(async () => { + await run(cwd, ["init", "-b", "main"]); + await run(cwd, ["config", "user.name", "Pylon Test"]); + await run(cwd, ["config", "user.email", "pylon@example.test"]); + await NodeFSP.writeFile( + NodePath.join(cwd, ".gitignore"), + ".private\nignored-target.txt\n", + ); + await NodeFSP.writeFile(NodePath.join(cwd, "tracked.txt"), "base\n"); + await NodeFSP.writeFile(NodePath.join(cwd, "deleted.txt"), "delete me\n"); + await NodeFSP.writeFile(NodePath.join(cwd, "rename-old.txt"), "rename me\n"); + await run(cwd, ["add", "-A"]); + await run(cwd, ["commit", "-m", "base"]); + + await NodeFSP.writeFile(NodePath.join(cwd, "tracked.txt"), "target\n"); + await NodeFSP.writeFile(NodePath.join(cwd, "target-added.txt"), "target added\n"); + await NodeFSP.writeFile( + NodePath.join(cwd, "ignored-target.txt"), + "target ignored path\n", + ); + await NodeFSP.rm(NodePath.join(cwd, "deleted.txt")); + await run(cwd, ["add", "-A"]); + await run(cwd, ["add", "-f", "ignored-target.txt"]); + const targetTree = await run(cwd, ["write-tree"]); + const targetOid = await run(cwd, [ + "commit-tree", + targetTree, + "-m", + "target checkpoint", + ]); + await run(cwd, ["update-ref", "refs/t3/checkpoints/thread-test/turn/1", targetOid]); + await run(cwd, ["reset", "--hard", "HEAD"]); + + await NodeFSP.writeFile(NodePath.join(cwd, "tracked.txt"), "staged\n"); + await run(cwd, ["add", "tracked.txt"]); + await NodeFSP.writeFile(NodePath.join(cwd, "tracked.txt"), "unstaged\n"); + await NodeFSP.rm(NodePath.join(cwd, "deleted.txt")); + await run(cwd, ["mv", "rename-old.txt", "rename-new.txt"]); + await NodeFSP.mkdir(NodePath.join(cwd, "nested")); + await NodeFSP.writeFile( + NodePath.join(cwd, "nested", "untracked.txt"), + "untracked canary\n", + ); + await NodeFSP.symlink("tracked.txt", NodePath.join(cwd, "link.txt")); + await NodeFSP.writeFile(NodePath.join(cwd, ".private"), "ignored private canary\n"); + await NodeFSP.writeFile( + NodePath.join(cwd, "ignored-target.txt"), + "source ignored path\n", + ); + await run(cwd, ["update-ref", "refs/t3/checkpoints/thread-test/turn/2", "HEAD"]); + }); + + const sourceStatus = yield* Effect.promise(() => + run(cwd, ["status", "--porcelain=v1", "-uall"]), + ); + const sourceIndex = yield* Effect.promise(() => run(cwd, ["ls-files", "--stage", "-z"])); + const sourceRefs = yield* Effect.promise(() => + run(cwd, ["for-each-ref", "--format=%(refname) %(objectname)", "refs/t3/checkpoints"]), + ); + const target = yield* workspace.resolveCheckpoint({ + cwd, + checkpointRef: "refs/t3/checkpoints/thread-test/turn/1", + }); + const preimage = yield* workspace.capturePreimage({ + operationId: "operation-workspace", + cwd, + targetCheckpointOid: target.oid, + }); + assert.notEqual(preimage.digest.length, 0); + assert.ok(!preimage.backupPath.startsWith(`${cwd}${NodePath.sep}`)); + const applied = yield* workspace.applyCheckpoint({ cwd, checkpointOid: target.oid }); + assert.notEqual(applied.digest, preimage.digest); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(cwd, "tracked.txt"), "utf8"), + ), + "target\n", + ); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(cwd, "target-added.txt"), "utf8"), + ), + "target added\n", + ); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(cwd, "ignored-target.txt"), "utf8"), + ), + "target ignored path\n", + ); + assert.isFalse( + yield* Effect.promise(() => + NodeFSP.stat(NodePath.join(cwd, "nested", "untracked.txt")).then( + () => true, + () => false, + ), + ), + ); + assert.isFalse( + yield* Effect.promise(() => + NodeFSP.stat(NodePath.join(cwd, "deleted.txt")).then( + () => true, + () => false, + ), + ), + ); + assert.equal( + yield* Effect.promise(() => NodeFSP.readFile(NodePath.join(cwd, ".private"), "utf8")), + "ignored private canary\n", + ); + assert.equal( + yield* Effect.promise(() => run(cwd, ["symbolic-ref", "HEAD"])), + "refs/heads/main", + ); + assert.equal( + yield* Effect.promise(() => + run(cwd, ["rev-parse", "refs/t3/checkpoints/thread-test/turn/2"]), + ), + yield* Effect.promise(() => run(cwd, ["rev-parse", "HEAD"])), + ); + + yield* Effect.promise(async () => { + await run(cwd, ["update-ref", "-d", "refs/t3/checkpoints/thread-test/turn/2"]); + await run(cwd, ["update-ref", "refs/t3/checkpoints/rogue/turn/99", "HEAD"]); + }); + const restored = yield* workspace.restorePreimage({ cwd, preimage }); + assert.equal(restored.digest, preimage.digest); + assert.equal( + yield* Effect.promise(() => run(cwd, ["status", "--porcelain=v1", "-uall"])), + sourceStatus, + ); + assert.equal( + yield* Effect.promise(() => run(cwd, ["ls-files", "--stage", "-z"])), + sourceIndex, + ); + assert.equal( + yield* Effect.promise(() => + run(cwd, [ + "for-each-ref", + "--format=%(refname) %(objectname)", + "refs/t3/checkpoints", + ]), + ), + sourceRefs, + ); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(cwd, "nested", "untracked.txt"), "utf8"), + ), + "untracked canary\n", + ); + assert.equal( + yield* Effect.promise(() => NodeFSP.readlink(NodePath.join(cwd, "link.txt"))), + "tracked.txt", + ); + assert.equal( + yield* Effect.promise(() => NodeFSP.readFile(NodePath.join(cwd, ".private"), "utf8")), + "ignored private canary\n", + ); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(cwd, "ignored-target.txt"), "utf8"), + ), + "source ignored path\n", + ); + + yield* workspace.cleanupPreimage(preimage); + assert.isFalse( + yield* Effect.promise(() => + NodeFSP.stat(preimage.backupPath).then( + () => true, + () => false, + ), + ), + ); + }), + (path) => Effect.promise(() => NodeFSP.rm(path, { recursive: true, force: true })), + ), + ); + + it.effect("uses distinct leases and restores the index for a linked worktree", () => + Effect.acquireUseRelease( + Effect.tryPromise(() => + NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-rollback-linked-")), + ), + (root) => + Effect.gen(function* () { + const workspace = yield* RollbackWorkspace; + const main = NodePath.join(root, "main"); + const linked = NodePath.join(root, "linked"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(main); + await run(main, ["init", "-b", "main"]); + await run(main, ["config", "user.name", "Pylon Test"]); + await run(main, ["config", "user.email", "pylon@example.test"]); + await NodeFSP.writeFile(NodePath.join(main, "tracked.txt"), "base\n"); + await run(main, ["add", "tracked.txt"]); + await run(main, ["commit", "-m", "base"]); + await run(main, ["worktree", "add", "-b", "linked-branch", linked]); + await NodeFSP.writeFile(NodePath.join(linked, "tracked.txt"), "checkpoint\n"); + await run(linked, ["add", "tracked.txt"]); + const tree = await run(linked, ["write-tree"]); + const checkpoint = await run(linked, ["commit-tree", tree, "-m", "linked checkpoint"]); + await run(linked, ["update-ref", "refs/t3/checkpoints/linked/turn/1", checkpoint]); + await run(linked, ["reset", "--hard", "HEAD"]); + await NodeFSP.writeFile(NodePath.join(linked, "tracked.txt"), "linked pre-image\n"); + await run(linked, ["add", "tracked.txt"]); + await NodeFSP.writeFile(NodePath.join(linked, "tracked.txt"), "linked unstaged\n"); + }); + + const mainIdentity = yield* workspace.resolveIdentity(main); + const linkedIdentity = yield* workspace.resolveIdentity(linked); + assert.equal(mainIdentity.gitCommonDir, linkedIdentity.gitCommonDir); + assert.notEqual(mainIdentity.workspaceKey, linkedIdentity.workspaceKey); + + const checkpoint = yield* workspace.resolveCheckpoint({ + cwd: linked, + checkpointRef: "refs/t3/checkpoints/linked/turn/1", + }); + const preimage = yield* workspace.capturePreimage({ + operationId: "operation-linked", + cwd: linked, + targetCheckpointOid: checkpoint.oid, + }); + yield* workspace.applyCheckpoint({ cwd: linked, checkpointOid: checkpoint.oid }); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(linked, "tracked.txt"), "utf8"), + ), + "checkpoint\n", + ); + const restored = yield* workspace.restorePreimage({ cwd: linked, preimage }); + assert.equal(restored.digest, preimage.digest); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(linked, "tracked.txt"), "utf8"), + ), + "linked unstaged\n", + ); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(main, "tracked.txt"), "utf8"), + ), + "base\n", + ); + yield* workspace.cleanupPreimage(preimage); + }), + (path) => Effect.promise(() => NodeFSP.rm(path, { recursive: true, force: true })), + ), + ); +}); diff --git a/apps/server/src/rollback/RollbackWorkspace.ts b/apps/server/src/rollback/RollbackWorkspace.ts new file mode 100644 index 000000000..0169af3b8 --- /dev/null +++ b/apps/server/src/rollback/RollbackWorkspace.ts @@ -0,0 +1,523 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeUtil from "node:util"; +import { ServerConfig } from "../config.ts"; + +const execFileAsync = NodeUtil.promisify(NodeChildProcess.execFile); +const MAX_PREIMAGE_ENTRIES = 20_000; +const MAX_PREIMAGE_BYTES = 512 * 1024 * 1024; + +export class RollbackWorkspaceError extends Schema.TaggedErrorClass()( + "RollbackWorkspaceError", + { code: Schema.String, cause: Schema.optional(Schema.Defect()) }, +) { + override get message(): string { + return `Rollback workspace operation failed (${this.code}).`; + } +} + +export interface RollbackWorkspaceIdentity { + readonly cwd: string; + readonly workspaceKey: string; + readonly gitCommonDir: string; +} +export interface RollbackCheckpointIdentity { + readonly oid: string; + readonly digest: string; +} +export interface RollbackWorkspacePreimage { + readonly backupPath: string; + readonly digest: string; + readonly indexPath: string; + readonly indexExisted: boolean; + readonly headSymbolic: string | null; + readonly headOid: string | null; + readonly ownedRefs: ReadonlyArray<{ readonly ref: string; readonly oid: string }>; + readonly paths: ReadonlyArray; + readonly entryCount: number; + readonly totalBytes: number; +} +export interface RollbackWorkspaceReceipt { + readonly digest: string; + readonly treeDigest: string; + readonly headSymbolic: string | null; + readonly headOid: string | null; +} + +export interface RollbackWorkspaceShape { + readonly resolveIdentity: ( + cwd: string, + ) => Effect.Effect; + readonly resolveCheckpoint: (input: { + readonly cwd: string; + readonly checkpointRef: string; + }) => Effect.Effect; + readonly capturePreimage: (input: { + readonly operationId: string; + readonly cwd: string; + readonly targetCheckpointOid: string; + }) => Effect.Effect; + readonly restorePreimage: (input: { + readonly cwd: string; + readonly preimage: RollbackWorkspacePreimage; + }) => Effect.Effect; + readonly applyCheckpoint: (input: { + readonly cwd: string; + readonly checkpointOid: string; + }) => Effect.Effect; + readonly inspect: ( + cwd: string, + ) => Effect.Effect; + readonly inspectCheckpoint: (input: { + readonly cwd: string; + readonly checkpointOid: string; + }) => Effect.Effect; + readonly cleanupPreimage: ( + preimage: RollbackWorkspacePreimage, + ) => Effect.Effect; +} +export class RollbackWorkspace extends Context.Service()( + "t3/rollback/RollbackWorkspace", +) {} + +async function git( + cwd: string, + args: ReadonlyArray, + allowFailure = false, + env?: Readonly>, +): Promise { + try { + const result = await execFileAsync("git", [...args], { + cwd, + timeout: 30_000, + maxBuffer: 16 * 1024 * 1024, + encoding: "utf8", + windowsHide: true, + ...(env === undefined ? {} : { env: { ...process.env, ...env } }), + }); + return result.stdout; + } catch (cause) { + if ( + allowFailure && + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === 1 + ) { + return ""; + } + throw cause; + } +} + +async function listMutablePaths(cwd: string): Promise> { + const raw = await git(cwd, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]); + return Array.from(new Set(raw.split("\0").filter(Boolean))).toSorted(); +} + +async function listCheckpointPaths( + cwd: string, + checkpointOid: string, +): Promise> { + const raw = await git(cwd, ["ls-tree", "-r", "--name-only", "-z", checkpointOid]); + return Array.from(new Set(raw.split("\0").filter(Boolean))).toSorted(); +} + +async function walkPaths( + root: string, + paths: ReadonlyArray, +): Promise<{ entries: number; bytes: number; digest: string }> { + const hash = NodeCrypto.createHash("sha256"); + let entries = 0; + let bytes = 0; + for (const relative of paths) { + const absolute = NodePath.join(root, relative); + const stat = await NodeFSP.lstat(absolute).catch((cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT") return null; + throw cause; + }); + entries += 1; + if (entries > MAX_PREIMAGE_ENTRIES) throw new Error("entry-bound"); + hash.update(relative); + hash.update("\0"); + if (stat === null) { + hash.update("missing\0"); + continue; + } + if (stat.isSymbolicLink()) { + const target = await NodeFSP.readlink(absolute); + hash.update("l\0"); + hash.update(target); + hash.update("\0"); + } else if (stat.isFile()) { + hash.update(String(stat.mode & 0o111)); + hash.update("\0"); + bytes += stat.size; + if (bytes > MAX_PREIMAGE_BYTES) throw new Error("byte-bound"); + hash.update("f\0"); + hash.update(String(stat.size)); + hash.update("\0"); + await new Promise((resolve, reject) => { + const stream = NodeFS.createReadStream(absolute); + stream.on("data", (chunk) => hash.update(chunk)); + stream.on("error", reject); + stream.on("end", resolve); + }); + hash.update("\0"); + } else { + throw new Error("unsupported-entry"); + } + } + return { entries, bytes, digest: hash.digest("hex") }; +} + +async function copyPaths( + source: string, + target: string, + paths: ReadonlyArray, +): Promise { + await NodeFSP.mkdir(target, { recursive: true, mode: 0o700 }); + for (const relative of paths) { + const sourcePath = NodePath.join(source, relative); + const exists = await NodeFSP.lstat(sourcePath).then( + () => true, + () => false, + ); + if (!exists) continue; + const targetPath = NodePath.join(target, relative); + await NodeFSP.mkdir(NodePath.dirname(targetPath), { recursive: true, mode: 0o700 }); + await NodeFSP.cp(sourcePath, targetPath, { + recursive: false, + dereference: false, + verbatimSymlinks: true, + preserveTimestamps: true, + errorOnExist: false, + force: true, + }); + } +} + +async function copyBackup(source: string, target: string): Promise { + await NodeFSP.mkdir(target, { recursive: true, mode: 0o700 }); + for (const name of await NodeFSP.readdir(source)) { + await NodeFSP.cp(NodePath.join(source, name), NodePath.join(target, name), { + recursive: true, + dereference: false, + verbatimSymlinks: true, + preserveTimestamps: true, + errorOnExist: false, + force: true, + }); + } +} + +async function clearMutablePaths( + cwd: string, + extraPaths: ReadonlyArray = [], +): Promise { + const paths = new Set([...(await listMutablePaths(cwd)), ...extraPaths]); + for (const relative of paths) { + await NodeFSP.rm(NodePath.join(cwd, relative), { recursive: true, force: true }); + } +} + +async function restoreCheckpointRefs( + cwd: string, + ownedRefs: ReadonlyArray<{ readonly ref: string; readonly oid: string }>, +): Promise { + const currentRaw = await git(cwd, [ + "for-each-ref", + "--format=%(refname)%00%(objectname)", + "refs/t3/checkpoints", + ]); + const currentRefs = currentRaw + .split("\n") + .filter(Boolean) + .map((line) => line.split("\0")[0] ?? ""); + const desired = new Map(ownedRefs.map((entry) => [entry.ref, entry.oid])); + const commands = [ + ...currentRefs.filter((ref) => !desired.has(ref)).map((ref) => `delete ${ref}`), + ...ownedRefs.map((entry) => `update ${entry.ref} ${entry.oid}`), + ]; + if (commands.length === 0) return; + await new Promise((resolve, reject) => { + const child = NodeChildProcess.execFile( + "git", + ["update-ref", "--stdin"], + { cwd, timeout: 30_000, windowsHide: true }, + (error) => { + if (error) reject(error); + else resolve(); + }, + ); + child.stdin?.end(`${commands.join("\n")}\n`); + }); +} + +async function resolveHead(cwd: string): Promise<{ symbolic: string | null; oid: string | null }> { + const symbolic = (await git(cwd, ["symbolic-ref", "-q", "HEAD"], true)).trim() || null; + const oid = + (await git(cwd, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"], true)).trim() || null; + return { symbolic, oid }; +} + +async function resolveIndexPath(cwd: string): Promise { + const raw = (await git(cwd, ["rev-parse", "--git-path", "index"])).trim(); + return NodePath.isAbsolute(raw) ? raw : NodePath.resolve(cwd, raw); +} + +async function resolveWorkspaceIdentity(cwd: string): Promise { + const top = (await git(cwd, ["rev-parse", "--show-toplevel"])).trim(); + const commonRaw = (await git(cwd, ["rev-parse", "--git-common-dir"])).trim(); + const canonicalTop = await NodeFSP.realpath(top); + const common = NodePath.isAbsolute(commonRaw) ? commonRaw : NodePath.resolve(cwd, commonRaw); + const canonicalCommon = await NodeFSP.realpath(common); + const workspaceKey = NodeCrypto.createHash("sha256") + .update(canonicalCommon) + .update("\0") + .update(canonicalTop) + .digest("hex"); + return { cwd: canonicalTop, workspaceKey, gitCommonDir: canonicalCommon }; +} + +async function writeWorkspaceTree( + cwd: string, + forcedPaths: ReadonlyArray, +): Promise { + const gitPathRaw = (await git(cwd, ["rev-parse", "--git-path", "t3"])).trim(); + const privateGitPath = NodePath.isAbsolute(gitPathRaw) + ? gitPathRaw + : NodePath.resolve(cwd, gitPathRaw); + await NodeFSP.mkdir(privateGitPath, { recursive: true, mode: 0o700 }); + const indexPath = NodePath.join( + privateGitPath, + `rollback-inspect-${NodeCrypto.randomUUID()}.index`, + ); + const env = { GIT_INDEX_FILE: indexPath }; + try { + await git(cwd, ["read-tree", "--empty"], false, env); + await git(cwd, ["add", "-A", "--", "."], false, env); + const existingForcedPaths: string[] = []; + for (const relative of forcedPaths) { + const exists = await NodeFSP.lstat(NodePath.join(cwd, relative)).then( + (stat) => stat.isFile() || stat.isSymbolicLink(), + () => false, + ); + if (exists) existingForcedPaths.push(relative); + } + if (existingForcedPaths.length > 0) { + await git(cwd, ["add", "-f", "--", ...existingForcedPaths], false, env); + } + return (await git(cwd, ["write-tree"], false, env)).trim(); + } finally { + await NodeFSP.rm(indexPath, { force: true }); + } +} + +async function inspectWorkspace( + cwd: string, + forcedPaths: ReadonlyArray = [], +): Promise { + const paths = Array.from(new Set([...(await listMutablePaths(cwd)), ...forcedPaths])).toSorted(); + const workspace = await walkPaths(cwd, paths); + const indexPath = await resolveIndexPath(cwd); + const hash = NodeCrypto.createHash("sha256"); + hash.update(workspace.digest); + if ( + await NodeFSP.stat(indexPath).then( + () => true, + () => false, + ) + ) + hash.update(await NodeFSP.readFile(indexPath)); + const head = await resolveHead(cwd); + const treeDigest = await writeWorkspaceTree(cwd, forcedPaths); + hash.update(head.symbolic ?? ""); + hash.update(head.oid ?? ""); + hash.update(treeDigest); + return { + digest: hash.digest("hex"), + treeDigest, + headSymbolic: head.symbolic, + headOid: head.oid, + }; +} + +const make = Effect.gen(function* () { + const config = yield* ServerConfig; + const run = (code: string, body: () => Promise) => + Effect.tryPromise({ try: body, catch: (cause) => new RollbackWorkspaceError({ code, cause }) }); + + const resolveIdentity: RollbackWorkspaceShape["resolveIdentity"] = (cwd) => + run("identity", () => resolveWorkspaceIdentity(cwd)); + + const resolveCheckpoint: RollbackWorkspaceShape["resolveCheckpoint"] = (input) => + run("checkpoint-identity", async () => { + const oid = ( + await git(input.cwd, ["rev-parse", "--verify", `${input.checkpointRef}^{commit}`]) + ).trim(); + const digest = (await git(input.cwd, ["rev-parse", "--verify", `${oid}^{tree}`])).trim(); + if (!/^[0-9a-f]{40,64}$/u.test(oid) || !/^[0-9a-f]{40,64}$/u.test(digest)) + throw new Error("invalid-oid"); + return { oid, digest }; + }); + + const capturePreimage: RollbackWorkspaceShape["capturePreimage"] = (input) => + run("preimage-capture", async () => { + const identity = await resolveWorkspaceIdentity(input.cwd); + const configuredBackupPath = NodePath.resolve( + config.baseDir, + "rollback-private", + input.operationId, + ); + const relativeBackupPath = NodePath.relative(identity.cwd, configuredBackupPath); + const backupPath = + relativeBackupPath !== "" && + !relativeBackupPath.startsWith(`..${NodePath.sep}`) && + relativeBackupPath !== ".." + ? NodePath.join(identity.gitCommonDir, "t3", "rollback-private", input.operationId) + : configuredBackupPath; + await NodeFSP.rm(backupPath, { recursive: true, force: true }); + await NodeFSP.mkdir(backupPath, { recursive: true, mode: 0o700 }); + const mutablePaths = Array.from( + new Set([ + ...(await listMutablePaths(identity.cwd)), + ...(await listCheckpointPaths(identity.cwd, input.targetCheckpointOid)), + ]), + ).toSorted(); + const measured = await walkPaths(identity.cwd, mutablePaths); + const workspaceBackup = NodePath.join(backupPath, "workspace"); + await copyPaths(identity.cwd, workspaceBackup, mutablePaths); + const copied = await walkPaths(workspaceBackup, mutablePaths); + if (copied.digest !== measured.digest) { + throw new Error("copy-mismatch"); + } + const indexPath = await resolveIndexPath(identity.cwd); + const indexExisted = await NodeFSP.stat(indexPath).then( + () => true, + () => false, + ); + if (indexExisted) await NodeFSP.copyFile(indexPath, NodePath.join(backupPath, "index")); + const head = await resolveHead(identity.cwd); + const refsRaw = await git(identity.cwd, [ + "for-each-ref", + "--format=%(refname)%00%(objectname)", + "refs/t3/checkpoints", + ]); + const ownedRefs = refsRaw + .split("\n") + .filter(Boolean) + .map((line) => { + const [ref = "", oid = ""] = line.split("\0"); + return { ref, oid }; + }); + const receipt = await inspectWorkspace(identity.cwd, mutablePaths); + return { + backupPath, + digest: receipt.digest, + indexPath, + indexExisted, + headSymbolic: head.symbolic, + headOid: head.oid, + ownedRefs, + paths: mutablePaths, + entryCount: measured.entries, + totalBytes: measured.bytes, + }; + }); + + const restorePreimage: RollbackWorkspaceShape["restorePreimage"] = (input) => + run("preimage-restore", async () => { + const identity = await resolveWorkspaceIdentity(input.cwd); + const beforeHead = await resolveHead(identity.cwd); + if ( + beforeHead.symbolic !== input.preimage.headSymbolic || + beforeHead.oid !== input.preimage.headOid + ) + throw new Error("head-drift"); + await clearMutablePaths(identity.cwd, input.preimage.paths); + await copyBackup(NodePath.join(input.preimage.backupPath, "workspace"), identity.cwd); + if (input.preimage.indexExisted) { + await NodeFSP.mkdir(NodePath.dirname(input.preimage.indexPath), { recursive: true }); + await NodeFSP.copyFile( + NodePath.join(input.preimage.backupPath, "index"), + input.preimage.indexPath, + ); + } else { + await NodeFSP.rm(input.preimage.indexPath, { force: true }); + } + await restoreCheckpointRefs(identity.cwd, input.preimage.ownedRefs); + const receipt = await inspectWorkspace(identity.cwd, input.preimage.paths); + if (receipt.digest !== input.preimage.digest) throw new Error("preimage-postcondition"); + return receipt; + }); + + const applyCheckpoint: RollbackWorkspaceShape["applyCheckpoint"] = (input) => + run("checkpoint-apply", async () => { + const identity = await resolveWorkspaceIdentity(input.cwd); + const beforeHead = await resolveHead(identity.cwd); + const targetPaths = await listCheckpointPaths(identity.cwd, input.checkpointOid); + const untracked = ( + await git(identity.cwd, ["ls-files", "--others", "--exclude-standard", "-z"]) + ) + .split("\0") + .filter(Boolean); + for (const relative of new Set([...untracked, ...targetPaths])) { + await NodeFSP.rm(NodePath.join(identity.cwd, relative), { recursive: true, force: true }); + } + await git(identity.cwd, [ + "restore", + "--source", + input.checkpointOid, + "--worktree", + "--staged", + "--", + ".", + ]); + if (beforeHead.oid !== null) await git(identity.cwd, ["reset", "--quiet", "--", "."]); + const receipt = await inspectWorkspace(identity.cwd, targetPaths); + const targetTree = ( + await git(identity.cwd, ["rev-parse", "--verify", `${input.checkpointOid}^{tree}`]) + ).trim(); + if (receipt.headSymbolic !== beforeHead.symbolic || receipt.headOid !== beforeHead.oid) + throw new Error("head-mutated"); + if (receipt.treeDigest !== targetTree) throw new Error("checkpoint-postcondition"); + return receipt; + }); + + const inspect: RollbackWorkspaceShape["inspect"] = (cwd) => + run("inspect", () => inspectWorkspace(cwd)); + const inspectCheckpoint: RollbackWorkspaceShape["inspectCheckpoint"] = (input) => + run("inspect-checkpoint", async () => { + const paths = await listCheckpointPaths(input.cwd, input.checkpointOid); + const receipt = await inspectWorkspace(input.cwd, paths); + const targetTree = ( + await git(input.cwd, ["rev-parse", "--verify", `${input.checkpointOid}^{tree}`]) + ).trim(); + if (receipt.treeDigest !== targetTree) throw new Error("checkpoint-postcondition"); + return receipt; + }); + const cleanupPreimage: RollbackWorkspaceShape["cleanupPreimage"] = (preimage) => + run("cleanup", () => NodeFSP.rm(preimage.backupPath, { recursive: true, force: true })); + + return RollbackWorkspace.of({ + resolveIdentity, + resolveCheckpoint, + capturePreimage, + restorePreimage, + applyCheckpoint, + inspect, + inspectCheckpoint, + cleanupPreimage, + }); +}); + +export const layer = Layer.effect(RollbackWorkspace, make); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 835d4eda9..bf0ba4afe 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -30,6 +30,7 @@ import { layerConfig as SqlitePersistenceLayerLive } from "./persistence/Layers/ import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import { ProviderSessionDirectoryLive } from "./provider/Layers/ProviderSessionDirectory.ts"; +import { RollbackSagaRepositoryLive } from "./persistence/Layers/RollbackSagas.ts"; import * as ProviderSessionRuntime from "./persistence/ProviderSessionRuntime.ts"; import * as PrimeAgentRecoveryLedger from "./provider/prime/PrimeAgentRecoveryLedger.ts"; import { ProviderAdapterRegistryLive } from "./provider/Layers/ProviderAdapterRegistry.ts"; @@ -63,6 +64,7 @@ import { ProviderRuntimeIngestionLive } from "./orchestration/Layers/ProviderRun import { ProviderCommandReactorLive } from "./orchestration/Layers/ProviderCommandReactor.ts"; import { CheckpointReactorLive } from "./orchestration/Layers/CheckpointReactor.ts"; import { ThreadDeletionReactorLive } from "./orchestration/Layers/ThreadDeletionReactor.ts"; +import * as RollbackSagaRunner from "./rollback/RollbackSagaRunner.ts"; import * as AgentAwarenessRelay from "./relay/AgentAwarenessRelay.ts"; import { hasCloudPublicConfig } from "./cloud/publicConfig.ts"; import { ProviderRegistryLive } from "./provider/Layers/ProviderRegistry.ts"; @@ -113,6 +115,8 @@ import * as ResourceMonitorBinary from "./resourceTelemetry/ResourceMonitorBinar import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts"; import * as UsageService from "./usage/UsageService.ts"; import { OrchestrationLayerLive } from "./orchestration/runtimeLayer.ts"; +import * as RollbackAdmission from "./rollback/RollbackAdmission.ts"; +import * as RollbackWorkspace from "./rollback/RollbackWorkspace.ts"; import { clearPersistedServerRuntimeState, makePersistedServerRuntimeState, @@ -252,11 +256,17 @@ const PlatformServicesLive = Layer.unwrap( }), ); +const RollbackSagaRuntimeLayerLive = RollbackSagaRunner.layer.pipe( + Layer.provideMerge(RollbackWorkspace.layer), + Layer.provideMerge(RollbackSagaRepositoryLive), +); + const ReactorLayerLive = Layer.empty.pipe( Layer.provideMerge(OrchestrationReactorLive), Layer.provideMerge(ProviderRuntimeIngestionLive), Layer.provideMerge(ProviderCommandReactorLive), Layer.provideMerge(CheckpointReactorLive), + Layer.provideMerge(RollbackSagaRuntimeLayerLive), Layer.provideMerge(ThreadDeletionReactorLive), Layer.provideMerge(AgentAwarenessRelay.layer.pipe(Layer.provide(ServerSecretStore.layer))), Layer.provideMerge(RuntimeReceiptBusLive), @@ -275,6 +285,7 @@ const ProviderSessionDirectoryLayerLive = ProviderSessionDirectoryLive.pipe( const ProviderLayerLive = ProviderServiceLive.pipe( Layer.provide(ProviderAdapterRegistryLive), Layer.provideMerge(ProviderSessionDirectoryLayerLive), + Layer.provideMerge(RollbackSagaRepositoryLive), ); const PersistenceLayerLive = PrimeAgentRecoveryLedger.layer.pipe( @@ -308,6 +319,7 @@ const GitLayerLive = Layer.empty.pipe( const GitWorkflowLayerLive = GitWorkflowService.layer.pipe( Layer.provideMerge(VcsDriverRegistryLayerLive), Layer.provideMerge(GitLayerLive), + Layer.provideMerge(RollbackSagaRepositoryLive), ); const SourceControlRepositoryServiceLayerLive = SourceControlRepositoryService.layer.pipe( @@ -378,9 +390,15 @@ const CloudManagedEndpointRuntimeLive = Layer.mergeAll( ), ); +const RollbackAdmissionLayerLive = RollbackAdmission.layer.pipe( + Layer.provide(ProviderLayerLive), + Layer.provide(RollbackWorkspace.layer), + Layer.provide(RollbackSagaRepositoryLive), +); + const ProviderRuntimeLayerLive = ProviderSessionReaperLive.pipe( Layer.provideMerge(ProviderLayerLive), - Layer.provideMerge(OrchestrationLayerLive), + Layer.provideMerge(OrchestrationLayerLive.pipe(Layer.provideMerge(RollbackAdmissionLayerLive))), ); const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe( diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 118a18479..b905da4a0 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -29,7 +29,6 @@ import { type OrchestrationEvent, type OrchestrationShellStreamEvent, type OrchestrationShellStreamItem, - type OrchestrationThreadStreamItem, OrchestrationGetFullThreadDiffError, OrchestrationGetSnapshotError, OrchestrationSearchThreadsError, @@ -322,6 +321,7 @@ export function isThreadDetailEvent(event: OrchestrationEvent): event is Extract | "thread.activity-appended" | "thread.turn-diff-completed" | "thread.reverted" + | "thread.rollback-status-updated" | "thread.session-set"; } > { @@ -331,6 +331,7 @@ export function isThreadDetailEvent(event: OrchestrationEvent): event is Extract event.type === "thread.activity-appended" || event.type === "thread.turn-diff-completed" || event.type === "thread.reverted" || + event.type === "thread.rollback-status-updated" || event.type === "thread.session-set" ); } diff --git a/docs/internals/glossary.md b/docs/internals/glossary.md index f98ee3051..326c052c1 100644 --- a/docs/internals/glossary.md +++ b/docs/internals/glossary.md @@ -158,6 +158,18 @@ The durable identifier for a filesystem checkpoint, stored as a Git ref. It is t The starting checkpoint for diffing a thread timeline. This flow is surfaced through [RuntimeReceiptBus.ts][13], coordinated in [CheckpointReactor.ts][6], and supported by [Utils.ts][22]. +#### Rollback saga + +A durable, provider-neutral operation that moves a checkpoint-managed workspace and an exact provider conversation anchor to the same older turn. Admission stores the operation and workspace lease before acknowledgement. The reconciler persists each phase, commits `thread.reverted` last, and either compensates to the source state or keeps the lease in manual recovery. Private state lives in [RollbackSagas.ts][29]; execution lives in [RollbackSagaRunner.ts][30]. + +#### Workspace rollback lease + +The durable fence for one canonical Git workspace while a rollback saga is active. It blocks new turns, provider input and queue mutations, checkpoint mutations, session stops, and Git mutations from every thread that shares the workspace. The lease is removed only after verified cleanup or proved compensation. + +#### Manual recovery + +A durable rollback state used when Pylon cannot prove that both the workspace and provider conversation are at either the target or compensated source. The public projection exposes only `manual-recovery`; exact anchors, receipts, paths, and workspace pre-images remain private. The workspace lease stays active until a later recovery path can prove a safe result. + #### Checkpoint diff The patch difference between two checkpoints. Query logic lives in [CheckpointDiffQuery.ts][20], diff parsing lives in [Diffs.ts][23], and finalization is coordinated by [CheckpointReactor.ts][6]. @@ -224,3 +236,5 @@ ships Pylon already matching it. [26]: ../../apps/server/src/orchestration/threadDetailCursor.ts [27]: ../../apps/server/src/environmentTheme.ts [28]: ../user/environment-theme.md +[29]: ../../apps/server/src/persistence/Services/RollbackSagas.ts +[30]: ../../apps/server/src/rollback/RollbackSagaRunner.ts diff --git a/docs/internals/providers.md b/docs/internals/providers.md index b14770ec1..37351c968 100644 --- a/docs/internals/providers.md +++ b/docs/internals/providers.md @@ -24,6 +24,12 @@ adapter in a child scope. Adapter implementations live beside them in [`ProviderAdapter.ts`][adapter]. Read the driver plus its adapter to see how a specific agent's transport, config, and event shapes are mapped. +### Absolute conversation rollback + +The optional `absoluteConversationRollback` adapter boundary captures, inspects, and applies a private JSON anchor with a stable equality digest. The durable rollback saga accepts only adapters that declare `conversationRollback: "absolute"` and implement all three operations. Relative turn counts and uninspectable no-op paths always fail closed. + +All built-in production adapters currently omit this boundary and remain `unsupported`. The foundation provides only the provider-neutral contract, private persistence, leases, compensation, and reconciliation. A later provider-specific phase must prove its immutable anchor semantics before changing that declaration. Anchors, native session identities, receipts, and recovery paths must not enter orchestration events, logs, shell projections, or client payloads. + Prime Agent uses its public detached-daemon APIs as the primary runtime on macOS, Linux, and WSL2 (which reports itself as Linux). `PrimeAgentDriver.create` rejects a native `win32` server before maintenance resolution, backend negotiation, status or catalog probes, capacity reads, adapter diff --git a/packages/client-runtime/src/state/threadReducer.test.ts b/packages/client-runtime/src/state/threadReducer.test.ts index a1e804de2..6328c1447 100644 --- a/packages/client-runtime/src/state/threadReducer.test.ts +++ b/packages/client-runtime/src/state/threadReducer.test.ts @@ -888,10 +888,39 @@ describe("applyThreadDetailEvent", () => { }); }); + describe("thread.rollback-status-updated", () => { + it("keeps pending and manual recovery durable in the shared client state", () => { + const pending = applyThreadDetailEvent(baseThread, { + ...baseEventFields, + sequence: 13, + occurredAt: "2026-04-01T03:30:00.000Z", + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-1"), + type: "thread.rollback-status-updated", + payload: { + threadId: ThreadId.make("thread-1"), + status: "manual-recovery", + updatedAt: "2026-04-01T03:30:00.000Z", + }, + }); + expect(pending.kind).toBe("updated"); + if (pending.kind === "updated") { + expect(pending.thread.rollbackStatus).toEqual({ + state: "manual-recovery", + updatedAt: "2026-04-01T03:30:00.000Z", + }); + } + }); + }); + describe("thread.reverted", () => { it("filters entities to retained turns", () => { const threadWithData: OrchestrationThread = { ...baseThread, + rollbackStatus: { + state: "pending", + updatedAt: "2026-04-01T03:30:00.000Z", + }, messages: [ { id: MessageId.make("msg-1"), @@ -964,6 +993,7 @@ describe("applyThreadDetailEvent", () => { // msg-3 (turn-2) is filtered, msg-1 (no turn) and msg-2 (turn-1) remain expect(result.thread.messages).toHaveLength(2); expect(result.thread.latestTurn?.turnId).toBe("turn-1"); + expect(result.thread.rollbackStatus).toBeNull(); } }); }); diff --git a/packages/client-runtime/src/state/threadReducer.ts b/packages/client-runtime/src/state/threadReducer.ts index 7b512c577..2e68f8ef7 100644 --- a/packages/client-runtime/src/state/threadReducer.ts +++ b/packages/client-runtime/src/state/threadReducer.ts @@ -525,6 +525,19 @@ export function applyThreadDetailEvent( } // ── Revert ────────────────────────────────────────────────────── + case "thread.rollback-status-updated": + return { + kind: "updated", + thread: { + ...thread, + rollbackStatus: + event.payload.status === null + ? null + : { state: event.payload.status, updatedAt: event.payload.updatedAt }, + updatedAt: event.payload.updatedAt, + }, + }; + case "thread.reverted": { const checkpoints = pipe( thread.checkpoints, @@ -556,6 +569,7 @@ export function applyThreadDetailEvent( messages, proposedPlans, activities, + rollbackStatus: null, latestTurn: latestCheckpoint === null ? null diff --git a/packages/contracts/src/orchestration.ts b/packages/contracts/src/orchestration.ts index 7e90b96e3..6fb0bb6de 100644 --- a/packages/contracts/src/orchestration.ts +++ b/packages/contracts/src/orchestration.ts @@ -585,6 +585,18 @@ export const ThreadLinkedPullRequest = Schema.Struct({ }); export type ThreadLinkedPullRequest = typeof ThreadLinkedPullRequest.Type; +export const OrchestrationRollbackPublicState = Schema.Literals([ + "pending", + "recovering", + "manual-recovery", +]); +export type OrchestrationRollbackPublicState = typeof OrchestrationRollbackPublicState.Type; +export const OrchestrationRollbackStatus = Schema.Struct({ + state: OrchestrationRollbackPublicState, + updatedAt: IsoDateTime, +}); +export type OrchestrationRollbackStatus = typeof OrchestrationRollbackStatus.Type; + export const OrchestrationThread = Schema.Struct({ id: ThreadId, projectId: ProjectId, @@ -598,6 +610,7 @@ export const OrchestrationThread = Schema.Struct({ worktreePath: Schema.NullOr(TrimmedNonEmptyString), linkedPullRequest: Schema.optional(Schema.NullOr(ThreadLinkedPullRequest)), latestTurn: Schema.NullOr(OrchestrationLatestTurn), + rollbackStatus: Schema.optional(Schema.NullOr(OrchestrationRollbackStatus)), createdAt: IsoDateTime, updatedAt: IsoDateTime, archivedAt: Schema.NullOr(IsoDateTime).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -685,6 +698,7 @@ export const OrchestrationThreadShell = Schema.Struct({ worktreePath: Schema.NullOr(TrimmedNonEmptyString), linkedPullRequest: Schema.optional(Schema.NullOr(ThreadLinkedPullRequest)), latestTurn: Schema.NullOr(OrchestrationLatestTurn), + rollbackStatus: Schema.optional(Schema.NullOr(OrchestrationRollbackStatus)), createdAt: IsoDateTime, updatedAt: IsoDateTime, archivedAt: Schema.NullOr(IsoDateTime).pipe(Schema.withDecodingDefault(Effect.succeed(null))), @@ -1171,6 +1185,8 @@ const ThreadCheckpointRevertCommand = Schema.Struct({ commandId: CommandId, threadId: ThreadId, turnCount: NonNegativeInt, + /** Exact current checkpoint revision. Required when an absolute adapter is admitted. */ + expectedSourceRevision: Schema.optional(NonNegativeInt), createdAt: IsoDateTime, }); @@ -1343,10 +1359,21 @@ const ThreadRevertCompleteCommand = Schema.Struct({ type: Schema.Literal("thread.revert.complete"), commandId: CommandId, threadId: ThreadId, + operationId: Schema.String, + sourceRevision: NonNegativeInt, + targetRevision: NonNegativeInt, turnCount: NonNegativeInt, createdAt: IsoDateTime, }); +const ThreadRollbackStatusSetCommand = Schema.Struct({ + type: Schema.Literal("thread.rollback.status.set"), + commandId: CommandId, + threadId: ThreadId, + status: Schema.NullOr(OrchestrationRollbackPublicState), + createdAt: IsoDateTime, +}); + const ThreadTurnAdmissionAcceptCommand = Schema.Struct({ type: Schema.Literal("thread.turn.admission.accept"), commandId: CommandId, @@ -1387,6 +1414,7 @@ const InternalOrchestrationCommand = Schema.Union([ ThreadTurnDiffCompleteCommand, ThreadActivityAppendCommand, ThreadRevertCompleteCommand, + ThreadRollbackStatusSetCommand, ThreadTurnAdmissionAcceptCommand, ThreadTurnAdmissionFailCommand, ThreadTitleRegenerationCompleteCommand, @@ -1425,6 +1453,7 @@ export const OrchestrationEventType = Schema.Literals([ "thread.user-input-response-requested", "thread.interaction-response-requested", "thread.checkpoint-revert-requested", + "thread.rollback-status-updated", "thread.reverted", "thread.session-stop-requested", "thread.session-set", @@ -1660,9 +1689,18 @@ export const ThreadCheckpointRevertRequestedPayload = Schema.Struct({ createdAt: IsoDateTime, }); +export const ThreadRollbackStatusUpdatedPayload = Schema.Struct({ + threadId: ThreadId, + status: Schema.NullOr(OrchestrationRollbackPublicState), + updatedAt: IsoDateTime, +}); + export const ThreadRevertedPayload = Schema.Struct({ threadId: ThreadId, turnCount: NonNegativeInt, + operationId: Schema.optional(Schema.String), + sourceRevision: Schema.optional(NonNegativeInt), + targetRevision: Schema.optional(NonNegativeInt), }); export const ThreadSessionStopRequestedPayload = Schema.Struct({ @@ -1866,6 +1904,11 @@ export const OrchestrationEvent = Schema.Union([ type: Schema.Literal("thread.checkpoint-revert-requested"), payload: ThreadCheckpointRevertRequestedPayload, }), + Schema.Struct({ + ...EventBaseFields, + type: Schema.Literal("thread.rollback-status-updated"), + payload: ThreadRollbackStatusUpdatedPayload, + }), Schema.Struct({ ...EventBaseFields, type: Schema.Literal("thread.reverted"),