diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 02ccb32cf..374f84507 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -125,6 +125,8 @@ jobs: packages/ui/src/stores/session-metadata.test.ts packages/ui/src/stores/session-pagination.test.ts packages/ui/src/stores/workspace-list-reconciliation-fence.test.ts + packages/ui/src/stores/workflows.test.ts + packages/opencode-plugin/plugin/lib/workflows.test.ts - name: Test restore ownership integration run: >- diff --git a/packages/electron-app/electron/main/client-state-cross-host.test.ts b/packages/electron-app/electron/main/client-state-cross-host.test.ts index 1d7e0b382..ef4ffc5e4 100644 --- a/packages/electron-app/electron/main/client-state-cross-host.test.ts +++ b/packages/electron-app/electron/main/client-state-cross-host.test.ts @@ -9,10 +9,12 @@ import test from "node:test" import { CrossHostRegistration, CROSS_HOST_OWNER_DIRECTORY, + crossHostParticipants, resolveCrossHostElectionDirectory, resolveCrossHostStatePath, type CrossHostLeaseDependencies, } from "./client-state-cross-host" +import { getMachineIdentity } from "./client-state-process-identity" import type { ProcessOwner } from "./client-state-process" function temp(t: test.TestContext): string { @@ -21,8 +23,8 @@ function temp(t: test.TestContext): string { return path } -function owner(pid: number, token: string, identity = `${token}-start`): ProcessOwner { - return { pid, runToken: token, processStartIdentity: identity } +function owner(pid: number, token: string, identity = `${token}-start`, machineIdentity = getMachineIdentity()!): ProcessOwner { + return { pid, runToken: token, processStartIdentity: identity, machineIdentity } } function dependencies(alive: boolean, identity?: string): CrossHostLeaseDependencies { @@ -141,6 +143,90 @@ test("simultaneous claimants deterministically recover a stale owner", (t) => { assert.equal(loser.isPrimary, false) }) +test("a separate machine with the same hostname is never probed or reclaimed", (t) => { + const directory = temp(t), remote = owner(201, "remote", "remote-start", "machine-b") + mkdirSync(join(directory, CROSS_HOST_OWNER_DIRECTORY)) + writeFileSync(ownerFile(directory), JSON.stringify({ ...remote, hostname: "shared-name" })) + let probes = 0 + const registration = CrossHostRegistration.register(directory, owner(202, "local"), true, { + pidAlive: () => { probes += 1; return false }, + processStartIdentity: () => { probes += 1; return "reused" }, + })! + assert.equal(registration.isPrimary, false) + assert.equal(probes, 0) + assert.equal(JSON.parse(readFileSync(ownerFile(directory), "utf8")).runToken, "remote") +}) + +test("legacy records fail closed unless their Linux boot marker proves locality", (t) => { + const remoteDirectory = temp(t) + mkdirSync(join(remoteDirectory, CROSS_HOST_OWNER_DIRECTORY)) + writeFileSync(ownerFile(remoteDirectory), JSON.stringify({ pid: 211, runToken: "legacy", processStartIdentity: "win32:old" })) + assert.equal(CrossHostRegistration.register(remoteDirectory, owner(212, "local"), true, dependencies(false))!.isPrimary, false) + + const localDirectory = temp(t) + mkdirSync(join(localDirectory, CROSS_HOST_OWNER_DIRECTORY)) + writeFileSync(ownerFile(localDirectory), JSON.stringify({ pid: 213, runToken: "legacy", processStartIdentity: "linux:boot-a:old" })) + assert.equal(CrossHostRegistration.register( + localDirectory, + owner(214, "local", "linux:boot-a:new"), + true, + dependencies(false), + )!.isPrimary, true) +}) + +test("a same-cohort non-candidate claims for the eligible local owner in either lexical order", async (t) => { + for (const [candidatePid, helperPid] of [[612, 613], [615, 614]]) { + const directory = temp(t), stale = owner(611, "tauri-stale", "tauri-start") + const candidateOwner = owner(candidatePid, "candidate"), helperOwner = owner(helperPid, "helper") + mkdirSync(join(directory, CROSS_HOST_OWNER_DIRECTORY)) + const observed = JSON.stringify(stale) + writeFileSync(ownerFile(directory), observed) + const identities = new Map([ + [candidateOwner.pid, candidateOwner.processStartIdentity], + [helperOwner.pid, helperOwner.processStartIdentity], + ]) + const deps = { + pidAlive: (pid: number) => identities.has(pid), + processStartIdentity: (pid: number) => identities.get(pid), + processStartIdentityAsync: async (pid: number) => identities.get(pid), + } + const helper = CrossHostRegistration.register(directory, helperOwner, false, deps)! + const candidate = CrossHostRegistration.register(directory, candidateOwner, true, deps)! + assert.equal(candidate.isPrimary, false) + + await helper.participateInRecoveryAsync(candidateOwner) + assert.equal(helper.isPrimary, false) + assert.equal(readFileSync(ownerFile(directory), "utf8"), observed) + assert.equal(existsSync(join(directory, `participant.${helperPid}.helper.json`)), false) + assert.equal(readFileSync(join(directory, `recovery.${candidatePid}.candidate.claim`), "utf8"), observed) + assert.equal(await candidate.tryAcquireAsync(true), true) + assert.equal(helper.isPrimary, false) + assert.equal(JSON.parse(readFileSync(ownerFile(directory), "utf8")).runToken, "candidate") + } +}) + +test("async recovery replaces a prior claim for a consecutive crashed owner", async (t) => { + const directory = temp(t), candidate = owner(620, "candidate") + const registration = CrossHostRegistration.register(directory, candidate, false, { + pidAlive: () => true, + processStartIdentity: () => { throw new Error("sync identity lookup must not run") }, + processStartIdentityAsync: async () => "reused-start", + })! + const publishStaleOwner = (stale: ProcessOwner) => { + mkdirSync(join(directory, CROSS_HOST_OWNER_DIRECTORY)) + writeFileSync(ownerFile(directory), JSON.stringify(stale)) + } + const first = owner(621, "first", "first-start") + publishStaleOwner(first) + assert.equal(await registration.tryAcquireAsync(true), true) + rmSync(join(directory, CROSS_HOST_OWNER_DIRECTORY), { recursive: true, force: true }) + + const second = owner(622, "second", "second-start") + publishStaleOwner(second) + assert.equal(await registration.tryAcquireAsync(true), true) + assert.equal(readFileSync(join(directory, "recovery.620.candidate.claim"), "utf8"), JSON.stringify(second)) +}) + test("graceful primary release allows a successor while a secondary remains", (t) => { const directory = temp(t) const primary = CrossHostRegistration.register(directory, owner(401, "primary"), true, dependencies(true, "primary-start"))! @@ -171,11 +257,15 @@ test("graceful handoff retires the old cohort so a crashed successor can recover CrossHostRegistration.register(directory, secondaryOwner, true, dependencies(true, "primary-start"))! primary.release() - assert.equal(readdirSync(directory).some((name) => name.startsWith("retired.")), false) + assert.equal(readdirSync(directory).some((name) => name.startsWith("retired.participant.")), true) assert.equal(JSON.parse(readFileSync(ownerFile(directory), "utf8")).runToken, "successor") assert.equal(existsSync(join(directory, "participant.423.successor.json")), false) assert.equal(existsSync(join(directory, "participant.425.late.json")), false) assert.equal(existsSync(malformed), false) + assert.deepEqual( + crossHostParticipants(directory).map(({ runToken }) => runToken).sort(), + ["late", "secondary", "successor"], + ) const claimantOwner = owner(424, "claimant"), identities = new Map([ [secondaryOwner.pid, secondaryOwner.processStartIdentity], diff --git a/packages/electron-app/electron/main/client-state-cross-host.ts b/packages/electron-app/electron/main/client-state-cross-host.ts index 33cbbd82d..f3fd6e110 100644 --- a/packages/electron-app/electron/main/client-state-cross-host.ts +++ b/packages/electron-app/electron/main/client-state-cross-host.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto" import { closeSync, existsSync, fsyncSync, linkSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, unlinkSync, writeFileSync } from "node:fs" import { homedir } from "node:os" import { basename, dirname, join, posix, win32 } from "node:path" -import { getProcessStartIdentity, type ProcessStartIdentityLookup } from "./client-state-process-identity" +import { getMachineIdentity, getProcessStartIdentity, getProcessStartIdentityAsync, type AsyncProcessStartIdentityLookup, type ProcessStartIdentityLookup } from "./client-state-process-identity" import { hasErrorCode, isPidAlive, type ProcessOwner } from "./client-state-process" export const CROSS_HOST_OWNER_DIRECTORY = "primary.owner.json" @@ -12,11 +12,14 @@ const PARTICIPANT_SUFFIX = ".json" const RECOVERY_PREFIX = "recovery." const RECOVERY_SUFFIX = ".claim" const RETIRED_PREFIX = "retired." +const RETIRED_PARTICIPANT_PREFIX = "retired.participant." +const RETIRED_PARTICIPANT_SUFFIX = ".json" const ACQUIRE_ATTEMPTS = 10 export interface CrossHostLeaseDependencies { pidAlive(pid: number): boolean processStartIdentity: ProcessStartIdentityLookup + processStartIdentityAsync?: AsyncProcessStartIdentityLookup onParticipantPublished?(): void onOwnerPrepared?(): void onOwnerRetired?(): void @@ -26,6 +29,7 @@ export interface CrossHostLeaseDependencies { const defaultDependencies: CrossHostLeaseDependencies = { pidAlive: isPidAlive, processStartIdentity: getProcessStartIdentity, + processStartIdentityAsync: getProcessStartIdentityAsync, } function validHome(value: string | undefined, platform: NodeJS.Platform): string | undefined { @@ -77,11 +81,14 @@ export function resolveLegacyTauriDataDirectory( export function createCrossHostOwner(): ProcessOwner | undefined { const processStartIdentity = getProcessStartIdentity(process.pid) - return processStartIdentity ? { pid: process.pid, runToken: randomUUID(), processStartIdentity } : undefined + const machineIdentity = getMachineIdentity() + return processStartIdentity && machineIdentity + ? { pid: process.pid, runToken: randomUUID(), processStartIdentity, machineIdentity } + : undefined } function serializeOwner(owner: ProcessOwner): string { - return JSON.stringify({ pid: owner.pid, runToken: owner.runToken, processStartIdentity: owner.processStartIdentity }) + return JSON.stringify({ pid: owner.pid, runToken: owner.runToken, processStartIdentity: owner.processStartIdentity, machineIdentity: owner.machineIdentity }) } function parseOwner(value: string): ProcessOwner | undefined { @@ -90,14 +97,20 @@ function parseOwner(value: string): ProcessOwner | undefined { if (Number.isInteger(owner.pid) && Number(owner.pid) > 0 && Number(owner.pid) <= 0xffff_ffff && typeof owner.runToken === "string" && /^[A-Za-z0-9_-]+$/.test(owner.runToken) && typeof owner.processStartIdentity === "string" && owner.processStartIdentity) { - return { pid: Number(owner.pid), runToken: owner.runToken, processStartIdentity: owner.processStartIdentity } + return { + pid: Number(owner.pid), + runToken: owner.runToken, + processStartIdentity: owner.processStartIdentity, + ...(typeof owner.machineIdentity === "string" && owner.machineIdentity ? { machineIdentity: owner.machineIdentity } : {}), + } } } catch {} return undefined } function sameOwner(left: ProcessOwner, right: ProcessOwner): boolean { - return left.pid === right.pid && left.runToken === right.runToken && left.processStartIdentity === right.processStartIdentity + return left.pid === right.pid && left.runToken === right.runToken && + left.processStartIdentity === right.processStartIdentity && left.machineIdentity === right.machineIdentity } function readIfExists(path: string): string | undefined { @@ -137,6 +150,10 @@ function recoveryPath(directory: string, owner: ProcessOwner): string { return join(directory, `${RECOVERY_PREFIX}${owner.pid}.${owner.runToken}${RECOVERY_SUFFIX}`) } +function retiredParticipantPath(directory: string, owner: ProcessOwner): string { + return join(directory, `${RETIRED_PARTICIPANT_PREFIX}${owner.pid}.${owner.runToken}${RETIRED_PARTICIPANT_SUFFIX}`) +} + function publishParticipant(path: string, owner: ProcessOwner): void { const value = serializeOwner(owner) try { publishFile(path, value) } catch (error) { @@ -144,16 +161,43 @@ function publishParticipant(path: string, owner: ProcessOwner): void { } } +function publishRecoveryClaim(path: string, observedOwner: string): void { + if (readIfExists(path) !== observedOwner) { + try { unlinkSync(path) } catch (error) { + if (!hasErrorCode(error, "ENOENT")) throw error + } + } + try { publishFile(path, observedOwner) } catch (error) { + if (!hasErrorCode(error, "EEXIST") || readIfExists(path) !== observedOwner) throw error + } +} + function ownerPath(directory: string): string { return join(directory, CROSS_HOST_OWNER_DIRECTORY, OWNER_FILENAME) } -function ownerIsStale(owner: ProcessOwner, dependencies: CrossHostLeaseDependencies): boolean | undefined { +function legacyOwnerIsLocal(owner: ProcessOwner, local: ProcessOwner): boolean { + const ownerBoot = /^(linux:[^:]+):/.exec(owner.processStartIdentity ?? "")?.[1] + const localBoot = /^(linux:[^:]+):/.exec(local.processStartIdentity ?? "")?.[1] + return Boolean(ownerBoot && ownerBoot === localBoot) +} + +function ownerIsStale(owner: ProcessOwner, local: ProcessOwner, dependencies: CrossHostLeaseDependencies): boolean | undefined { + if (owner.machineIdentity ? owner.machineIdentity !== local.machineIdentity : !legacyOwnerIsLocal(owner, local)) return undefined if (!dependencies.pidAlive(owner.pid)) return true const identity = dependencies.processStartIdentity(owner.pid) return identity ? identity !== owner.processStartIdentity : undefined } +async function ownerIsStaleAsync(owner: ProcessOwner, local: ProcessOwner, dependencies: CrossHostLeaseDependencies): Promise { + if (owner.machineIdentity ? owner.machineIdentity !== local.machineIdentity : !legacyOwnerIsLocal(owner, local)) return undefined + if (!dependencies.pidAlive(owner.pid)) return true + const identity = dependencies.processStartIdentityAsync + ? await dependencies.processStartIdentityAsync(owner.pid, 1_000) + : dependencies.processStartIdentity(owner.pid) + return identity ? identity !== owner.processStartIdentity : undefined +} + function removeParticipantIfOwned(path: string, owner: ProcessOwner): void { const observed = readIfExists(path) const current = observed === undefined ? undefined : parseOwner(observed) @@ -163,6 +207,10 @@ function removeParticipantIfOwned(path: string, owner: ProcessOwner): void { } } +function removeRetiredParticipantIfOwned(directory: string, owner: ProcessOwner): void { + removeParticipantIfOwned(retiredParticipantPath(directory, owner), owner) +} + function retireOwnerIfOwned(directory: string, owner: ProcessOwner, dependencies: CrossHostLeaseDependencies): void { const observed = readIfExists(ownerPath(directory)) const current = parseOwner(observed ?? "") @@ -182,6 +230,7 @@ function retireOwnerIfOwned(directory: string, owner: ProcessOwner, dependencies if (observedParticipant === undefined) continue const participant = parseOwner(observedParticipant) if (participant) { + publishParticipant(retiredParticipantPath(directory, participant), participant) removeParticipantIfOwned(path, participant) try { unlinkSync(recoveryPath(directory, participant)) } catch {} } else if (readIfExists(path) === observedParticipant) { @@ -208,7 +257,7 @@ function recoveryClaimants( const participant = parseOwner(readIfExists(path) ?? "") if (!participant) return undefined if (sameOwner(participant, current)) continue - const stale = ownerIsStale(participant, dependencies) + const stale = ownerIsStale(participant, current, dependencies) if (stale === true) { removeParticipantIfOwned(path, participant) try { unlinkSync(recoveryPath(directory, participant)) } catch {} @@ -226,8 +275,38 @@ function recoveryClaimants( return claimants } +async function recoveryClaimantsAsync( + directory: string, + current: ProcessOwner, + observedOwner: string, + dependencies: CrossHostLeaseDependencies, +): Promise { + const claimants = [current] + for (const name of readdirSync(directory)) { + if (!name.startsWith(PARTICIPANT_PREFIX) || !name.endsWith(PARTICIPANT_SUFFIX)) continue + const path = join(directory, name) + const participant = parseOwner(readIfExists(path) ?? "") + if (!participant) return undefined + if (sameOwner(participant, current)) continue + if (await ownerIsStaleAsync(participant, current, dependencies) === true) { + removeParticipantIfOwned(path, participant) + try { unlinkSync(recoveryPath(directory, participant)) } catch {} + continue + } + const claimPath = recoveryPath(directory, participant) + let claim = readIfExists(claimPath) + for (let attempt = 0; claim !== observedOwner && attempt < 20; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + claim = readIfExists(claimPath) + } + if (claim !== observedOwner) return undefined + claimants.push(participant) + } + return claimants +} + function retireOwner(directory: string, observed: string, owner: ProcessOwner, claimant: ProcessOwner, dependencies: CrossHostLeaseDependencies): boolean { - if (ownerIsStale(owner, dependencies) !== true) return false + if (ownerIsStale(owner, claimant, dependencies) !== true) return false const claimants = recoveryClaimants(directory, claimant, observed, dependencies) if (!claimants) return false claimants.sort((left, right) => serializeOwner(left) < serializeOwner(right) ? -1 : 1) @@ -244,6 +323,24 @@ function retireOwner(directory: string, observed: string, owner: ProcessOwner, c } } +async function retireOwnerAsync(directory: string, observed: string, owner: ProcessOwner, claimant: ProcessOwner, dependencies: CrossHostLeaseDependencies): Promise { + if (await ownerIsStaleAsync(owner, claimant, dependencies) !== true) return false + const claimants = await recoveryClaimantsAsync(directory, claimant, observed, dependencies) + if (!claimants) return false + claimants.sort((left, right) => serializeOwner(left) < serializeOwner(right) ? -1 : 1) + if (!sameOwner(claimants[0]!, claimant)) return false + if (readIfExists(ownerPath(directory)) !== observed) return false + const retired = join(directory, `${RETIRED_PREFIX}${owner.pid}.${owner.runToken}`) + try { + renameSync(join(directory, CROSS_HOST_OWNER_DIRECTORY), retired) + dependencies.onOwnerRetired?.() + return true + } catch (error) { + if (["ENOENT", "EEXIST", "ENOTEMPTY"].some((code) => hasErrorCode(error, code)) || existsSync(retired)) return false + throw error + } +} + function publishOwner(directory: string, owner: ProcessOwner, dependencies: CrossHostLeaseDependencies): boolean { const temporary = join(directory, `.owner.${randomUUID()}.tmp`) try { @@ -264,7 +361,10 @@ function publishOwner(directory: string, owner: ProcessOwner, dependencies: Cros export function crossHostParticipants(directory: string): ProcessOwner[] { try { return readdirSync(directory) - .filter((name) => name.startsWith(PARTICIPANT_PREFIX) && name.endsWith(PARTICIPANT_SUFFIX)) + .filter((name) => + (name.startsWith(PARTICIPANT_PREFIX) && name.endsWith(PARTICIPANT_SUFFIX)) || + (name.startsWith(RETIRED_PARTICIPANT_PREFIX) && name.endsWith(RETIRED_PARTICIPANT_SUFFIX)), + ) .map((name) => parseOwner(readIfExists(join(directory, name)) ?? "")) .filter((owner): owner is ProcessOwner => Boolean(owner)) } catch (error) { @@ -280,7 +380,7 @@ export class CrossHostRegistration { private readonly directory: string, readonly owner: ProcessOwner, private readonly participant: string, - private readonly recoveryClaim: string | undefined, + private recoveryClaim: string | undefined, private primary: boolean, private readonly dependencies: CrossHostLeaseDependencies, ) {} @@ -293,10 +393,11 @@ export class CrossHostRegistration { primaryCandidate: boolean | (() => boolean), dependencies: CrossHostLeaseDependencies = defaultDependencies, ): CrossHostRegistration | undefined { - if (!owner.processStartIdentity || !/^[A-Za-z0-9_-]+$/.test(owner.runToken)) return undefined + if (!owner.processStartIdentity || !owner.machineIdentity || !/^[A-Za-z0-9_-]+$/.test(owner.runToken)) return undefined mkdirSync(directory, { recursive: true, mode: 0o700 }) const participant = participantPath(directory, owner) publishParticipant(participant, owner) + removeRetiredParticipantIfOwned(directory, owner) dependencies.onParticipantPublished?.() let primary = false let recoveryClaim: string | undefined @@ -309,11 +410,9 @@ export class CrossHostRegistration { const existing = parseOwner(observed) if (!existing) break if (sameOwner(existing, owner)) { primary = true; break } - if (ownerIsStale(existing, dependencies) === true) { + if (ownerIsStale(existing, owner, dependencies) === true) { recoveryClaim ??= recoveryPath(directory, owner) - try { publishFile(recoveryClaim, observed) } catch (error) { - if (!hasErrorCode(error, "EEXIST") || readIfExists(recoveryClaim) !== observed) throw error - } + publishRecoveryClaim(recoveryClaim, observed) } if (!retireOwner(directory, observed, existing, owner, dependencies)) break } @@ -332,10 +431,79 @@ export class CrossHostRegistration { return Boolean(current && sameOwner(current, this.owner)) } + tryAcquire(primaryCandidate: boolean | (() => boolean)): boolean { + if (this.released || this.isPrimary) return this.isPrimary + if (!(typeof primaryCandidate === "function" ? primaryCandidate() : primaryCandidate)) return false + publishParticipant(this.participant, this.owner) + removeRetiredParticipantIfOwned(this.directory, this.owner) + for (let attempt = 0; attempt < ACQUIRE_ATTEMPTS; attempt += 1) { + if (publishOwner(this.directory, this.owner, this.dependencies)) { this.primary = true; break } + const observed = readIfExists(ownerPath(this.directory)) + if (observed === undefined) continue + const existing = parseOwner(observed) + if (!existing) break + if (sameOwner(existing, this.owner)) { this.primary = true; break } + if (ownerIsStale(existing, this.owner, this.dependencies) === true) { + this.recoveryClaim ??= recoveryPath(this.directory, this.owner) + publishRecoveryClaim(this.recoveryClaim, observed) + } + if (!retireOwner(this.directory, observed, existing, this.owner, this.dependencies)) break + } + return this.isPrimary + } + + async tryAcquireAsync(primaryCandidate: boolean | (() => boolean)): Promise { + if (this.released || this.isPrimary) return this.isPrimary + if (!(typeof primaryCandidate === "function" ? primaryCandidate() : primaryCandidate)) return false + publishParticipant(this.participant, this.owner) + removeRetiredParticipantIfOwned(this.directory, this.owner) + for (let attempt = 0; attempt < ACQUIRE_ATTEMPTS; attempt += 1) { + if (publishOwner(this.directory, this.owner, this.dependencies)) { this.primary = true; break } + const observed = readIfExists(ownerPath(this.directory)) + if (observed === undefined) continue + const existing = parseOwner(observed) + if (!existing) break + if (sameOwner(existing, this.owner)) { this.primary = true; break } + if (await ownerIsStaleAsync(existing, this.owner, this.dependencies) === true) { + this.recoveryClaim ??= recoveryPath(this.directory, this.owner) + publishRecoveryClaim(this.recoveryClaim, observed) + } + if (!await retireOwnerAsync(this.directory, observed, existing, this.owner, this.dependencies)) break + } + return this.isPrimary + } + + async participateInRecoveryAsync(candidate: ProcessOwner): Promise { + if (this.released) return + publishParticipant(this.participant, this.owner) + removeRetiredParticipantIfOwned(this.directory, this.owner) + const observed = readIfExists(ownerPath(this.directory)) + if (observed === undefined) return + const existing = parseOwner(observed) + if (!existing || await ownerIsStaleAsync(existing, this.owner, this.dependencies) !== true) return + if (await ownerIsStaleAsync(candidate, this.owner, this.dependencies) !== false) return + if (sameOwner(candidate, this.owner)) { + this.recoveryClaim ??= recoveryPath(this.directory, this.owner) + publishRecoveryClaim(this.recoveryClaim, observed) + return + } + removeParticipantIfOwned(this.participant, this.owner) + if (this.recoveryClaim) try { unlinkSync(this.recoveryClaim) } catch {} + this.recoveryClaim = undefined + publishRecoveryClaim(recoveryPath(this.directory, candidate), observed) + } + + deferPrimary(): void { + if (this.released) return + retireOwnerIfOwned(this.directory, this.owner, this.dependencies) + this.primary = false + } + release(): boolean { if (this.released) return false retireOwnerIfOwned(this.directory, this.owner, this.dependencies) removeParticipantIfOwned(this.participant, this.owner) + removeRetiredParticipantIfOwned(this.directory, this.owner) if (this.recoveryClaim) try { unlinkSync(this.recoveryClaim) } catch {} this.primary = false this.released = true diff --git a/packages/electron-app/electron/main/client-state-process-identity.ts b/packages/electron-app/electron/main/client-state-process-identity.ts index 749bfc57e..bf5cbcafa 100644 --- a/packages/electron-app/electron/main/client-state-process-identity.ts +++ b/packages/electron-app/electron/main/client-state-process-identity.ts @@ -1,11 +1,12 @@ import { execFile, spawnSync } from "node:child_process" -import { readFile as readFileAsync } from "node:fs/promises" +import { readFile as readFileAsync, readlink as readlinkAsync } from "node:fs/promises" import { readFileSync, readlinkSync } from "node:fs" import { basename, resolve } from "node:path" export type ProcessStartIdentityLookup = (pid: number) => string | undefined export type AsyncProcessStartIdentityLookup = (pid: number, timeoutMs: number) => Promise | string | undefined export type ExpectedProcessLookup = (pid: number) => boolean | undefined +export type AsyncExpectedProcessLookup = (pid: number, timeoutMs: number) => Promise | boolean | undefined function readLinuxProcessStartIdentity(pid: number): string | undefined { const stat = readFileSync(`/proc/${pid}/stat`, "utf8") @@ -127,3 +128,69 @@ export function isExpectedTauriProcess(pid: number): boolean | undefined { return undefined } } + +const machineIdentityCache = new Map() + +export function getMachineIdentity(platform: NodeJS.Platform = process.platform): string | undefined { + const cached = machineIdentityCache.get(platform) + if (cached !== undefined) return cached ?? undefined + const identity = readMachineIdentity(platform) + machineIdentityCache.set(platform, identity ?? null) + return identity +} + +function readMachineIdentity(platform: NodeJS.Platform): string | undefined { + try { + if (platform === "linux") { + for (const path of ["/etc/machine-id", "/var/lib/dbus/machine-id"]) { + try { + const value = readFileSync(path, "utf8").trim().toLowerCase() + if (value) return `linux:${value}` + } catch {} + } + } + if (platform === "darwin") { + const value = readCommandIdentity("ioreg", ["-rd1", "-c", "IOPlatformExpertDevice"], "ioreg") + const uuid = /"IOPlatformUUID"\s*=\s*"([^"]+)"/.exec(value ?? "")?.[1]?.toLowerCase() + return uuid ? `darwin:${uuid}` : undefined + } + if (platform === "win32") { + const value = readCommandIdentity("powershell.exe", [ + "-NoProfile", + "-NonInteractive", + "-Command", + "(Get-ItemProperty -LiteralPath 'HKLM:\\SOFTWARE\\Microsoft\\Cryptography' -Name MachineGuid -ErrorAction Stop).MachineGuid.ToString().ToLowerInvariant()", + ], "machine")?.slice("machine:".length) + return value ? `win32:${value}` : undefined + } + } catch { + // Cross-host ownership is disabled when the local machine cannot be identified safely. + } + return undefined +} + +export async function isExpectedTauriProcessAsync( + pid: number, + timeoutMs: number, + platform: NodeJS.Platform = process.platform, +): Promise { + if (!Number.isInteger(pid) || pid <= 0 || timeoutMs <= 0) return undefined + try { + const executable = platform === "linux" + ? await readlinkAsync(`/proc/${pid}/exe`) + : await readCommandIdentityAsync( + platform === "win32" ? "powershell.exe" : "ps", + platform === "win32" + ? ["-NoProfile", "-NonInteractive", "-Command", `(Get-Process -Id ${pid} -ErrorAction Stop).Path`] + : ["-p", String(pid), "-o", "comm="], + "path", + timeoutMs, + ).then((value) => value?.slice(5)) + if (!executable) return undefined + if (resolve(executable).toLowerCase() === resolve(process.execPath).toLowerCase()) return false + return ["codenomad", "codenomad.exe", "codenomad-tauri", "codenomad-tauri.exe"] + .includes(basename(executable).toLowerCase()) + } catch { + return undefined + } +} diff --git a/packages/electron-app/electron/main/client-state-process.test.ts b/packages/electron-app/electron/main/client-state-process.test.ts index fe9e0bde9..83918e2cb 100644 --- a/packages/electron-app/electron/main/client-state-process.test.ts +++ b/packages/electron-app/electron/main/client-state-process.test.ts @@ -165,6 +165,29 @@ test("a surviving older secondary keeps later processes secondary", async (t) => assert.equal(third.role.isPrimary, false) }) +test("a retained secondary retries after the local primary exits", (t) => { + const directory = temp(t) + const primaryLockPath = join(directory, "client-state.primary.lock") + const registrationLockPath = join(directory, "client-state.registration.lock") + const paths = { primaryLockPath, registrationLockPath } + const identities = new Map([[31, "first-start"], [32, "second-start"], [33, "third-start"]]) + const alive = (pid: number) => identities.has(pid) + const identity = (pid: number) => identities.get(pid) + const first = { pid: 31, runToken: "first", processStartIdentity: "first-start" } + const second = { pid: 32, runToken: "second", processStartIdentity: "second-start" } + const third = { pid: 33, runToken: "third", processStartIdentity: "third-start" } + + assert.equal(electClientStateProcess(directory, first, paths, () => {}, alive, 0, () => {}, identity), true) + assert.equal(electClientStateProcess(directory, second, paths, () => {}, alive, 0, () => {}, identity), false) + assert.equal(electClientStateProcess(directory, third, paths, () => {}, alive, 0, () => {}, identity), false) + removeProcessOwnerLockIfOwned(primaryLockPath, first) + removeRunningMarkerIfOwned(getRunningMarkerPath(directory, first), first) + identities.delete(first.pid) + + assert.equal(electClientStateProcess(directory, second, paths, () => {}, alive, 0, () => {}, identity, true), true) + assert.equal(electClientStateProcess(directory, third, paths, () => {}, alive, 0, () => {}, identity, true), false) +}) + test("lock recovery handles PID reuse, malformed files, and verified live owners", async (t) => { const cases = [ { name: "same PID old token", owner: { pid: process.pid, runToken: "new" }, file: { pid: process.pid, runToken: "old" }, lock: "primary", alive: () => true, expected: true }, diff --git a/packages/electron-app/electron/main/client-state-process.ts b/packages/electron-app/electron/main/client-state-process.ts index 6874dde03..2c5acf721 100644 --- a/packages/electron-app/electron/main/client-state-process.ts +++ b/packages/electron-app/electron/main/client-state-process.ts @@ -1,9 +1,13 @@ import { closeSync, fsyncSync, openSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from "node:fs" import { basename, join } from "node:path" import { + type AsyncExpectedProcessLookup, + type AsyncProcessStartIdentityLookup, type ExpectedProcessLookup, getProcessStartIdentity, + getProcessStartIdentityAsync, isExpectedTauriProcess, + isExpectedTauriProcessAsync, type ProcessStartIdentityLookup, } from "./client-state-process-identity" @@ -17,6 +21,7 @@ export interface ProcessOwner { pid: number runToken: string processStartIdentity?: string + machineIdentity?: string } export type RunningMarkerStatus = "current" | "other-live" | "stale" @@ -63,6 +68,9 @@ function normalizeProcessOwner(candidate: unknown): ProcessOwner | undefined { ...(typeof owner.processStartIdentity === "string" && owner.processStartIdentity ? { processStartIdentity: owner.processStartIdentity } : {}), + ...(typeof owner.machineIdentity === "string" && owner.machineIdentity + ? { machineIdentity: owner.machineIdentity } + : {}), } } return undefined @@ -118,6 +126,33 @@ export function hasLiveTauriClient( }) } +export async function hasLiveTauriClientAsync( + tauriDataPath: string, + pidAlive: (pid: number) => boolean = isPidAlive, + processStartIdentity: AsyncProcessStartIdentityLookup = getProcessStartIdentityAsync, + expectedProcess: AsyncExpectedProcessLookup = isExpectedTauriProcessAsync, + upgradedParticipants: readonly ProcessOwner[] = [], + timeoutMs = 1_000, +): Promise { + let entries: string[] + try { + entries = readdirSync(tauriDataPath) + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return false + throw error + } + for (const name of entries) { + const match = /^client-state\.running\.(\d+)\..+\.lock$/.exec(name) + if (!match) continue + const pid = Number(match[1]) + if (!Number.isInteger(pid) || pid <= 0 || !pidAlive(pid)) continue + const liveIdentity = await processStartIdentity(pid, timeoutMs) + if (liveIdentity && upgradedParticipants.some((owner) => owner.pid === pid && owner.processStartIdentity === liveIdentity)) continue + if (await expectedProcess(pid, timeoutMs) !== false) return true + } + return false +} + export function classifyRunningMarker( markerOwner: ProcessOwner, currentOwner: ProcessOwner, @@ -171,6 +206,25 @@ export function createRunningMarker( return markerPath } +function ensureRunningMarker( + userDataPath: string, + owner: ProcessOwner, + primaryOwner?: ProcessOwner, +): string { + try { + return createRunningMarker(userDataPath, owner, primaryOwner) + } catch (error) { + if (!hasErrorCode(error, "EEXIST")) throw error + const markerPath = getRunningMarkerPath(userDataPath, owner) + const existing = readFileIfExists(markerPath) + const existingOwner = existing === undefined ? undefined : parseProcessOwner(existing) + if (existingOwner && isSameProcessOwner(existingOwner, owner)) { + return markerPath + } + throw error + } +} + function publishProcessFile(path: string, contents: string): void { let descriptor: number | undefined try { @@ -313,6 +367,11 @@ export function isProcessOwnerLockOwned(path: string, owner: ProcessOwner): bool return Boolean(current && isSameProcessOwner(current, owner)) } +export function readProcessOwnerLock(path: string): ProcessOwner | undefined { + const value = readFileIfExists(path) + return value === undefined ? undefined : parseProcessOwner(value) +} + export function removeRunningMarkerIfOwned(markerPath: string, owner: ProcessOwner): boolean { const filenameOwner = parseRunningMarkerFilename(basename(markerPath)) if (!filenameOwner || !isSameProcessOwner(filenameOwner, owner)) { @@ -436,6 +495,7 @@ export function electClientStateProcess( registrationLockWaitMs = REGISTRATION_LOCK_WAIT_MS, onPrimaryLockAcquired: () => void = () => {}, processStartIdentity: ProcessStartIdentityLookup = getProcessStartIdentity, + retainedCandidate = false, ): boolean { let registrationAcquired = false let registeringOwner: ProcessOwner | undefined @@ -457,7 +517,8 @@ export function electClientStateProcess( if (!registrationAcquired) { try { - createRunningMarker(userDataPath, owner, registeringOwner) + if (retainedCandidate) ensureRunningMarker(userDataPath, owner, registeringOwner) + else createRunningMarker(userDataPath, owner, registeringOwner) } catch (error) { onWarning("failed to create running marker", error) } @@ -485,7 +546,7 @@ export function electClientStateProcess( if (isPrimary) { try { onPrimaryLockAcquired() - if (cleanStaleRunningMarkers(userDataPath, owner, pidAlive, processStartIdentity)) { + if (cleanStaleRunningMarkers(userDataPath, owner, pidAlive, processStartIdentity) && !retainedCandidate) { removeProcessOwnerLockIfOwned(paths.primaryLockPath, owner) isPrimary = false } @@ -497,7 +558,8 @@ export function electClientStateProcess( } try { - createRunningMarker(userDataPath, owner, acknowledgedPrimary) + if (retainedCandidate) ensureRunningMarker(userDataPath, owner, acknowledgedPrimary) + else createRunningMarker(userDataPath, owner, acknowledgedPrimary) } catch (error) { onWarning("failed to create running marker", error) if (isPrimary) releaseProcessOwnerLock(paths.primaryLockPath, owner, onWarning, "failed to release primary lock") diff --git a/packages/electron-app/electron/main/client-state.test.ts b/packages/electron-app/electron/main/client-state.test.ts index 18ee8d3ab..a8a6b3f1a 100644 --- a/packages/electron-app/electron/main/client-state.test.ts +++ b/packages/electron-app/electron/main/client-state.test.ts @@ -5,6 +5,10 @@ import { tmpdir } from "node:os" import { join } from "node:path" import test from "node:test" import { ClientStateManager, type ClientStateWriter } from "./client-state" +import { createRunningMarker } from "./client-state-process" +import { getMachineIdentity, getProcessStartIdentity } from "./client-state-process-identity" + +const MACHINE_IDENTITY = getMachineIdentity()! function harness(t: test.TestContext, initial?: object) { const directory = mkdtempSync(join(tmpdir(), "codenomad-state-")) @@ -17,7 +21,7 @@ function harness(t: test.TestContext, initial?: object) { writes++ if (failing) throw new Error("injected write failure") await writeFile(path, value, "utf8") - }, processOwner?: { pid: number; runToken: string; processStartIdentity: string }) => { + }, processOwner?: { pid: number; runToken: string; processStartIdentity: string; machineIdentity?: string }) => { const manager = new ClientStateManager(directory, writer, { crossHostElectionDirectory: join(directory, "election"), processOwner }) managers.push(manager) return manager @@ -58,7 +62,7 @@ test("cross-host ownership is required in addition to each host-local election", const primary = new ClientStateManager(tauriDirectory, undefined, { crossHostElectionDirectory: election, crossHostDependencies, - processOwner: { pid: 8101, runToken: "tauri", processStartIdentity: "tauri-start" }, + processOwner: { pid: 8101, runToken: "tauri", processStartIdentity: "tauri-start", machineIdentity: MACHINE_IDENTITY }, }) mkdirSync(electronDirectory) writeFileSync(join(electronDirectory, "client-state.json"), JSON.stringify({ @@ -69,7 +73,7 @@ test("cross-host ownership is required in addition to each host-local election", const secondary = new ClientStateManager(electronDirectory, undefined, { crossHostElectionDirectory: election, crossHostDependencies, - processOwner: { pid: 8102, runToken: "electron", processStartIdentity: "electron-start" }, + processOwner: { pid: 8102, runToken: "electron", processStartIdentity: "electron-start", machineIdentity: MACHINE_IDENTITY }, }) assert.deepEqual(secondary.loadClientState(), { isPrimary: false, restoreEnabled: false, snapshot: null }) await secondary.drainAndReleasePrimary() @@ -79,12 +83,156 @@ test("cross-host ownership is required in addition to each host-local election", const successor = new ClientStateManager(electronDirectory, undefined, { crossHostElectionDirectory: election, crossHostDependencies, - processOwner: { pid: 8103, runToken: "successor", processStartIdentity: "successor-start" }, + processOwner: { pid: 8103, runToken: "successor", processStartIdentity: "successor-start", machineIdentity: MACHINE_IDENTITY }, }) assert.equal(successor.isPrimary, true) await successor.drainAndReleasePrimary() }) +test("a retained secondary promotes after the other host exits and reloads before writes", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-late-host-")) + const firstDirectory = join(root, "first"), electronDirectory = join(root, "electron"), election = join(root, "election") + mkdirSync(firstDirectory); mkdirSync(electronDirectory) + t.after(() => rmSync(root, { recursive: true, force: true })) + const identities = new Map([[8201, "tauri-start"], [8202, "electron-start"]]) + const crossHostDependencies = { pidAlive: (pid: number) => identities.has(pid), processStartIdentity: (pid: number) => identities.get(pid) } + const first = new ClientStateManager(firstDirectory, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8201, runToken: "tauri", processStartIdentity: "tauri-start", machineIdentity: MACHINE_IDENTITY }, + }) + const secondary = new ClientStateManager(electronDirectory, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8202, runToken: "electron", processStartIdentity: "electron-start", machineIdentity: MACHINE_IDENTITY }, + }) + let notifications = 0 + secondary.onOwnershipChanged(() => { notifications += 1 }) + assert.equal(await secondary.refreshPrimary(), false) + assert.deepEqual(secondary.loadClientState(), { isPrimary: false, restoreEnabled: false, snapshot: null }) + const authoritative = { + version: 1, + restoreEnabled: true, + snapshot: { revision: 7, kept: true }, + window: { bounds: { x: 10, y: 20, width: 1200, height: 800 }, maximized: false, fullscreen: false, zoomFactor: 1.25 }, + } + writeFileSync(join(root, "client-state.json"), JSON.stringify(authoritative)) + + await first.drainAndReleasePrimary() + identities.delete(8201) + assert.equal(await secondary.refreshPrimary(), true) + assert.equal(notifications, 1) + assert.equal(await secondary.saveClientState({ revision: 1, default: true }), false) + assert.deepEqual(secondary.loadClientState().snapshot, authoritative.snapshot) + assert.equal(await secondary.saveClientState({ revision: 8, kept: true }), true) + assert.deepEqual(JSON.parse(readFileSync(join(root, "client-state.json"), "utf8")), { + ...authoritative, + snapshot: { revision: 8, kept: true }, + }) + await secondary.drainAndReleasePrimary() +}) + +test("a retained host-local secondary retries election after its primary exits", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-local-promotion-")) + const userData = join(root, "electron"), election = join(root, "election") + mkdirSync(userData) + t.after(() => rmSync(root, { recursive: true, force: true })) + const identities = new Map([[8231, "primary-start"], [8232, "secondary-start"]]) + const crossHostDependencies = { pidAlive: (pid: number) => identities.has(pid), processStartIdentity: (pid: number) => identities.get(pid) } + const primary = new ClientStateManager(userData, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8231, runToken: "primary", processStartIdentity: "primary-start", machineIdentity: MACHINE_IDENTITY }, + }) + const secondary = new ClientStateManager(userData, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8232, runToken: "secondary", processStartIdentity: "secondary-start", machineIdentity: MACHINE_IDENTITY }, + }) + assert.equal(secondary.isPrimary, false) + + await primary.drainAndReleasePrimary() + identities.delete(8231) + assert.equal(await secondary.refreshPrimary(), true) + assert.equal(secondary.isPrimary, true) + await secondary.drainAndReleasePrimary() +}) + +test("one eligible Electron recovers a stale Tauri owner while its local secondary stays fenced", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-local-stale-promotion-")) + const userData = join(root, "electron"), election = join(root, "election") + mkdirSync(userData); mkdirSync(join(election, "primary.owner.json"), { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const staleOwner = { pid: 8241, runToken: "tauri-stale", processStartIdentity: "tauri-start", machineIdentity: MACHINE_IDENTITY } + const localOwner = { pid: process.pid, runToken: "local", processStartIdentity: getProcessStartIdentity(process.pid), machineIdentity: MACHINE_IDENTITY } + const secondaryOwner = { pid: 8243, runToken: "secondary", processStartIdentity: "secondary-start", machineIdentity: MACHINE_IDENTITY } + writeFileSync(join(election, "primary.owner.json", "owner.json"), JSON.stringify(staleOwner)) + writeFileSync(join(userData, "client-state.primary.lock"), JSON.stringify(localOwner)) + const localMarker = createRunningMarker(userData, localOwner) + const identities = new Map([ + [localOwner.pid, localOwner.processStartIdentity], + [secondaryOwner.pid, secondaryOwner.processStartIdentity], + ]) + const crossHostDependencies = { + pidAlive: (pid: number) => identities.has(pid), + processStartIdentity: (pid: number) => identities.get(pid), + processStartIdentityAsync: async (pid: number) => identities.get(pid), + } + const secondary = new ClientStateManager(userData, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: secondaryOwner, + }) + assert.equal(await secondary.refreshPrimary(), false) + assert.equal(existsSync(join(election, "recovery.8243.secondary.claim")), false) + assert.equal(readFileSync(join(election, `recovery.${localOwner.pid}.local.claim`), "utf8"), JSON.stringify(staleOwner)) + + rmSync(join(userData, "client-state.primary.lock")); rmSync(localMarker) + const local = new ClientStateManager(userData, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: localOwner, + }) + assert.deepEqual([local.isPrimary, secondary.isPrimary], [true, false]) + assert.equal(await secondary.refreshPrimary(), false) + assert.equal(await secondary.saveClientState({ forbidden: true }), false) + assert.equal(await local.saveClientState({ recovered: true }), true) + assert.deepEqual(JSON.parse(readFileSync(join(root, "client-state.json"), "utf8")).snapshot, { recovered: true }) + await Promise.all([local.drainAndReleasePrimary(), secondary.drainAndReleasePrimary()]) +}) + +test("late promotion migrates legacy state when shared state is absent", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-late-migration-")) + const firstDirectory = join(root, "first"), electronDirectory = join(root, "electron"), election = join(root, "shared", "election") + mkdirSync(firstDirectory, { recursive: true }); mkdirSync(electronDirectory, { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const identities = new Map([[8251, "first-start"], [8252, "electron-start"]]) + const crossHostDependencies = { pidAlive: (pid: number) => identities.has(pid), processStartIdentity: (pid: number) => identities.get(pid) } + const first = new ClientStateManager(firstDirectory, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8251, runToken: "first", processStartIdentity: "first-start", machineIdentity: MACHINE_IDENTITY }, + }) + writeFileSync(join(electronDirectory, "client-state.json"), JSON.stringify({ + version: 1, + restoreEnabled: true, + snapshot: { revision: 5, savedAt: 42, legacy: true }, + })) + const secondary = new ClientStateManager(electronDirectory, undefined, { + crossHostElectionDirectory: election, + crossHostDependencies, + processOwner: { pid: 8252, runToken: "electron", processStartIdentity: "electron-start", machineIdentity: MACHINE_IDENTITY }, + }) + + await first.drainAndReleasePrimary() + identities.delete(8251) + assert.equal(await secondary.refreshPrimary(), true) + assert.deepEqual(secondary.loadClientState().snapshot, { revision: 5, savedAt: 42, legacy: true }) + assert.equal(existsSync(join(electronDirectory, "client-state.json")), false) + assert.equal(existsSync(join(root, "shared", "client-state.json")), true) + await secondary.drainAndReleasePrimary() +}) + test("first shared primary deterministically migrates legacy host envelopes", async (t) => { const root = mkdtempSync(join(tmpdir(), "codenomad-migration-")) const electron = join(root, "electron"), tauri = join(root, "tauri"), election = join(root, "shared", "election") @@ -105,16 +253,22 @@ test("first shared primary deterministically migrates legacy host envelopes", as await manager.drainAndReleasePrimary() }) -test("legacy migration prefers disabled and ignores malformed candidates", async (t) => { +test("legacy migration prefers disabled, strips stale payloads, and ignores malformed candidates", async (t) => { const root = mkdtempSync(join(tmpdir(), "codenomad-migration-")) const electron = join(root, "electron"), tauri = join(root, "tauri"), election = join(root, "shared", "election") mkdirSync(electron, { recursive: true }); mkdirSync(tauri, { recursive: true }) t.after(() => rmSync(root, { recursive: true, force: true })) writeFileSync(join(electron, "client-state.json"), "malformed") - writeFileSync(join(tauri, "client-state.json"), JSON.stringify({ version: 1, restoreEnabled: false, snapshot: { savedAt: 1 } })) + writeFileSync(join(tauri, "client-state.json"), JSON.stringify({ + version: 1, + restoreEnabled: false, + snapshot: { savedAt: 1 }, + window: { bounds: { x: 1, y: 2, width: 800, height: 600 }, maximized: false, fullscreen: false, zoomFactor: 1 }, + })) const manager = new ClientStateManager(electron, undefined, { crossHostElectionDirectory: election, legacyTauriDataPath: tauri }) assert.deepEqual(manager.loadClientState(), { isPrimary: true, restoreEnabled: false, snapshot: null }) - assert.equal(JSON.parse(readFileSync(join(root, "shared", "client-state.json"), "utf8")).restoreEnabled, false) + assert.deepEqual(JSON.parse(readFileSync(join(root, "shared", "client-state.json"), "utf8")), { version: 1, restoreEnabled: false }) + assert.equal(existsSync(join(tauri, "client-state.json")), false) await manager.drainAndReleasePrimary() }) @@ -143,6 +297,101 @@ test("legacy cleanup failure cannot abort startup after shared state replacement }) assert.deepEqual(manager.loadClientState().snapshot, { savedAt: 10 }) assert.equal(existsSync(join(shared, "client-state.json")), true) + assert.equal(existsSync(join(electron, "client-state.json")), true) + await manager.drainAndReleasePrimary() +}) + +test("legacy cleanup preserves a same-content file replaced after the migration snapshot", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-migration-race-")) + const electron = join(root, "electron"), tauri = join(root, "tauri"), election = join(root, "shared", "election") + mkdirSync(electron, { recursive: true }); mkdirSync(tauri, { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const contents = JSON.stringify({ version: 1, restoreEnabled: true, snapshot: { savedAt: 10 } }) + const electronLegacy = join(electron, "client-state.json"), tauriLegacy = join(tauri, "client-state.json") + writeFileSync(electronLegacy, contents); writeFileSync(tauriLegacy, contents) + + const manager = new ClientStateManager(electron, undefined, { + crossHostElectionDirectory: election, + legacyTauriDataPath: tauri, + removeLegacyState: (path) => { + rmSync(path, { force: true }) + if (path.includes(".electron.migration-quarantine")) { + rmSync(tauriLegacy) + writeFileSync(tauriLegacy, contents) + } + }, + }) + assert.equal(existsSync(electronLegacy), false) + assert.equal(readFileSync(tauriLegacy, "utf8"), contents) + await manager.drainAndReleasePrimary() +}) + +test("legacy cleanup does not delete a replacement created after quarantine", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-migration-quarantine-race-")) + const electron = join(root, "electron"), election = join(root, "shared", "election") + mkdirSync(electron, { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const legacy = join(electron, "client-state.json") + const replacement = JSON.stringify({ version: 1, restoreEnabled: true, snapshot: { replacement: true } }) + writeFileSync(legacy, JSON.stringify({ version: 1, restoreEnabled: true, snapshot: { savedAt: 10 } })) + + const manager = new ClientStateManager(electron, undefined, { + crossHostElectionDirectory: election, + removeLegacyState: (quarantinePath) => { + writeFileSync(legacy, replacement) + rmSync(quarantinePath, { force: true }) + }, + }) + assert.equal(readFileSync(legacy, "utf8"), replacement) + await manager.drainAndReleasePrimary() +}) + +test("legacy cleanup stops when a legacy Tauri host appears after replacement", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-migration-live-host-")) + const electron = join(root, "electron"), tauri = join(root, "tauri"), election = join(root, "shared", "election") + mkdirSync(electron, { recursive: true }); mkdirSync(tauri, { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const electronLegacy = join(electron, "client-state.json"), tauriLegacy = join(tauri, "client-state.json") + writeFileSync(electronLegacy, JSON.stringify({ version: 1, restoreEnabled: true })) + writeFileSync(tauriLegacy, JSON.stringify({ version: 1, restoreEnabled: true, snapshot: { savedAt: 10 } })) + + const manager = new ClientStateManager(electron, undefined, { + crossHostElectionDirectory: election, + legacyTauriDataPath: tauri, + crossHostDependencies: { pidAlive: (pid) => pid === 9912, processStartIdentity: () => undefined }, + removeLegacyState: (path) => { + rmSync(path, { force: true }) + if (path.includes(".electron.migration-quarantine")) writeFileSync(join(tauri, "client-state.running.9912.late.lock"), "") + }, + }) + assert.equal(existsSync(electronLegacy), false) + assert.equal(existsSync(tauriLegacy), true) + await manager.drainAndReleasePrimary() +}) + +test("legacy migration rechecks a Tauri host that starts after ownership acquisition", async (t) => { + const root = mkdtempSync(join(tmpdir(), "codenomad-migration-race-")) + const electron = join(root, "electron"), tauri = join(root, "tauri"), shared = join(root, "shared"), election = join(shared, "election") + mkdirSync(electron, { recursive: true }); mkdirSync(tauri, { recursive: true }); mkdirSync(shared, { recursive: true }) + t.after(() => rmSync(root, { recursive: true, force: true })) + const sharedState = join(shared, "client-state.json") + writeFileSync(sharedState, JSON.stringify({ version: 1, restoreEnabled: true })) + const manager = new ClientStateManager(electron, undefined, { + crossHostElectionDirectory: election, + legacyTauriDataPath: tauri, + crossHostDependencies: { pidAlive: (pid) => pid === 9911, processStartIdentity: () => undefined }, + }) + rmSync(sharedState) + const legacy = join(tauri, "client-state.json") + writeFileSync(legacy, JSON.stringify({ version: 1, restoreEnabled: true, snapshot: { legacy: true } })) + writeFileSync(join(tauri, "client-state.running.9911.late.lock"), "") + + assert.throws( + () => (manager as unknown as { migrateLegacyStateIfNeeded(): void }).migrateLegacyStateIfNeeded(), + /ownership changed before atomic replacement/, + ) + assert.equal(existsSync(sharedState), false) + assert.equal(existsSync(legacy), true) await manager.drainAndReleasePrimary() }) @@ -154,8 +403,11 @@ test("ownership loss immediately disables restore reads and mutations", async (t window: { width: 900, height: 700 }, }) const manager = h.create() + let notifications = 0 + manager.onOwnershipChanged(() => { notifications += 1 }) writeFileSync(join(h.directory, "election", "primary.owner.json", "owner.json"), "malformed") assert.equal(manager.isPrimary, false) + assert.equal(notifications, 1) assert.deepEqual(manager.loadClientState(), { isPrimary: false, restoreEnabled: false, snapshot: null }) assert.equal(manager.getWindowState(), undefined) assert.equal(await manager.saveClientState({ ignored: true }), false) @@ -193,7 +445,7 @@ test("successful clear suppresses saves, including after failed re-enable", asyn test("disabling restore atomically removes snapshot/window and survives restart", async (t) => { const h = harness(t, { version: 1, restoreEnabled: true }) - const manager = h.create(undefined, { pid: process.pid, runToken: "before-restart", processStartIdentity: "old-start" }) + const manager = h.create(undefined, { pid: process.pid, runToken: "before-restart", processStartIdentity: "old-start", machineIdentity: MACHINE_IDENTITY }) await manager.saveClientState({ kept: true }) await manager.saveWindowState({ bounds: { x: 10, y: 20, width: 1200, height: 800 }, maximized: true, fullscreen: false, zoomFactor: 1.25 }) const before = h.writes() @@ -238,7 +490,7 @@ test("an old writer cannot replace a successor after PID reuse", async (t) => { const gate = new Promise((resolve) => { release = resolve }) const old = h.create( async (path, value) => { await writeFile(path, value); started(); await gate }, - { pid: process.pid, runToken: "old-run", processStartIdentity: "old-start" }, + { pid: process.pid, runToken: "old-run", processStartIdentity: "old-start", machineIdentity: MACHINE_IDENTITY }, ) const staleWrite = old.saveClientState({ stale: true }) await began @@ -252,10 +504,41 @@ test("an old writer cannot replace a successor after PID reuse", async (t) => { assert.deepEqual(JSON.parse(readFileSync(h.statePath, "utf8")).snapshot, { successor: true }) }) +test("a write admitted before ownership loss cannot replace state after reacquisition", async (t) => { + const h = harness(t, { version: 1, restoreEnabled: true, snapshot: { initial: true } }) + let started!: () => void + let release!: () => void + const began = new Promise((resolve) => { started = resolve }) + const gate = new Promise((resolve) => { release = resolve }) + const manager = h.create(async (path, value) => { await writeFile(path, value); started(); await gate }) + let notifications = 0 + manager.onOwnershipChanged(() => { notifications += 1 }) + const staleWrite = manager.saveClientState({ stale: true }) + await began + + try { + rmSync(join(h.directory, "election", "primary.owner.json"), { recursive: true }) + assert.equal(manager.isPrimary, false) + assert.equal(notifications, 1) + assert.equal(await manager.refreshPrimary(), true) + assert.equal(notifications, 2) + assert.equal(await manager.saveClientState({ unreconciled: true }), false) + const successor = { version: 1, restoreEnabled: true, snapshot: { successor: true } } + writeFileSync(h.statePath, JSON.stringify(successor)) + } finally { + release() + } + + await assert.rejects(staleWrite, /ownership changed before atomic replacement/) + const successor = { version: 1, restoreEnabled: true, snapshot: { successor: true } } + assert.deepEqual(JSON.parse(readFileSync(h.statePath, "utf8")), successor) + assert.deepEqual(manager.loadClientState().snapshot, successor.snapshot) +}) + test("future envelopes are preserved until a successful explicit clear", async (t) => { const future = { version: 7, restoreEnabled: false, snapshot: { future: true }, futurePreference: "keep" } const h = harness(t, future) - const manager = h.create(undefined, { pid: process.pid, runToken: "future-before-restart", processStartIdentity: "old-start" }) + const manager = h.create(undefined, { pid: process.pid, runToken: "future-before-restart", processStartIdentity: "old-start", machineIdentity: MACHINE_IDENTITY }) assert.deepEqual(manager.loadClientState(), { isPrimary: true, restoreEnabled: false, snapshot: null }) assert.equal(await manager.saveClientState({ ignored: true }), true) assert.equal(await manager.setRestoreEnabled(false), false) diff --git a/packages/electron-app/electron/main/client-state.ts b/packages/electron-app/electron/main/client-state.ts index 98b09320b..6d715b9c2 100644 --- a/packages/electron-app/electron/main/client-state.ts +++ b/packages/electron-app/electron/main/client-state.ts @@ -1,18 +1,20 @@ import { randomUUID } from "node:crypto" -import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" +import { closeSync, fstatSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs" import { open, rename, rm } from "node:fs/promises" import { dirname, join } from "node:path" import { electClientStateProcess, getRunningMarkerPath, hasLiveTauriClient, + hasLiveTauriClientAsync, hasErrorCode, isProcessOwnerLockOwned, + readProcessOwnerLock, removeProcessOwnerLockIfOwned, type ProcessOwner, removeRunningMarkerIfOwned, } from "./client-state-process" -import { getProcessStartIdentity } from "./client-state-process-identity" +import { getMachineIdentity, getProcessStartIdentity } from "./client-state-process-identity" import { CrossHostRegistration, crossHostParticipants, @@ -114,23 +116,56 @@ function parseClientState(value: string): ParsedClientState { } } -function legacyCandidate(path: string, host: "electron" | "tauri"): { host: string; state: PersistedClientState; savedAt: number; hasSnapshot: boolean } | undefined { +interface LegacyStateSnapshot { + path: string + host: "electron" | "tauri" + contents: Buffer + identity: string +} + +function snapshotLegacyState(path: string, host: LegacyStateSnapshot["host"]): LegacyStateSnapshot | undefined { + let descriptor: number | undefined try { - const candidate = JSON.parse(readFileSync(path, "utf8")) as Record + descriptor = openSync(path, "r") + const before = legacyFileIdentity(descriptor) + const contents = readFileSync(descriptor) + const after = legacyFileIdentity(descriptor) + return before === after ? { path, host, contents, identity: after } : undefined + } catch { + return undefined + } finally { + if (descriptor !== undefined) closeSync(descriptor) + } +} + +function legacyFileIdentity(descriptor: number): string { + const value = fstatSync(descriptor, { bigint: true }) + return `${value.dev}:${value.ino}:${value.size}:${value.mtimeNs}:${value.birthtimeNs}` +} + +function legacySnapshotMatches(snapshot: LegacyStateSnapshot, path = snapshot.path): boolean { + const current = snapshotLegacyState(path, snapshot.host) + return Boolean(current && current.identity === snapshot.identity && current.contents.equals(snapshot.contents)) +} + +function legacyCandidate(snapshot: LegacyStateSnapshot): { host: string; state: PersistedClientState; savedAt: number; hasSnapshot: boolean } | undefined { + try { + const candidate = JSON.parse(snapshot.contents.toString("utf8")) as Record if (!candidate || candidate.version !== CLIENT_STATE_VERSION) return undefined const parsed = parseClientState(JSON.stringify(candidate)).state delete parsed.window - const snapshot = candidate.snapshot as Record | undefined - const savedAt = typeof snapshot?.savedAt === "number" && Number.isFinite(snapshot.savedAt) ? snapshot.savedAt : -1 - return { host, state: parsed, savedAt, hasSnapshot: snapshot !== undefined } + if (!parsed.restoreEnabled) delete parsed.snapshot + const candidateSnapshot = candidate.snapshot as Record | undefined + const savedAt = typeof candidateSnapshot?.savedAt === "number" && Number.isFinite(candidateSnapshot.savedAt) ? candidateSnapshot.savedAt : -1 + return { host: snapshot.host, state: parsed, savedAt, hasSnapshot: candidateSnapshot !== undefined } } catch { return undefined } } -function isFutureLegacyCandidate(path: string): boolean { +function isFutureLegacyCandidate(snapshot: LegacyStateSnapshot): boolean { try { - const candidate = JSON.parse(readFileSync(path, "utf8")) as Record + const candidate = JSON.parse(snapshot.contents.toString("utf8")) as Record return typeof candidate?.version === "number" && candidate.version > CLIENT_STATE_VERSION } catch { return false @@ -141,7 +176,10 @@ export class ClientStateManager { private readonly userDataPath: string private readonly statePath: string private readonly lockPath: string + private readonly registrationLockPath: string private readonly legacyTauriDataPath: string | null + private readonly legacyPaths: ReadonlyArray + private readonly removeLegacyState: (path: string) => void private readonly owner: ProcessOwner private state: PersistedClientState = { version: CLIENT_STATE_VERSION, restoreEnabled: true } private writeQueue: Promise = Promise.resolve() @@ -152,6 +190,11 @@ export class ClientStateManager { private unsupportedFutureEnvelope = false private frozen = false private rendererAccessToken: string | undefined + private rendererReconciliationPending = false + private effectivePrimary = false + private ownershipEpoch = 0 + private readonly ownershipListeners = new Set<() => void>() + private readonly crossHostDependencies: CrossHostLeaseDependencies | undefined constructor( userDataPath: string, @@ -162,7 +205,9 @@ export class ClientStateManager { pid: process.pid, runToken: randomUUID(), processStartIdentity: getProcessStartIdentity(process.pid), + machineIdentity: getMachineIdentity(), } + this.crossHostDependencies = options?.crossHostDependencies mkdirSync(userDataPath, { recursive: true }) this.userDataPath = userDataPath const crossHostElectionDirectory = options?.crossHostElectionDirectory ?? resolveCrossHostElectionDirectory() @@ -171,18 +216,23 @@ export class ClientStateManager { : resolveCrossHostStatePath() mkdirSync(dirname(this.statePath), { recursive: true }) this.lockPath = join(userDataPath, PRIMARY_LOCK_FILENAME) - const registrationLockPath = join(userDataPath, REGISTRATION_LOCK_FILENAME) + this.registrationLockPath = join(userDataPath, REGISTRATION_LOCK_FILENAME) const election = electClientStateProcess( userDataPath, this.owner, - { primaryLockPath: this.lockPath, registrationLockPath }, + { primaryLockPath: this.lockPath, registrationLockPath: this.registrationLockPath }, (message, error) => console.warn(`[client-state] ${message}`, error), ) const legacyTauriDataPath = options?.legacyTauriDataPath === undefined ? (options?.crossHostElectionDirectory ? null : resolveLegacyTauriDataDirectory()) : options.legacyTauriDataPath this.legacyTauriDataPath = legacyTauriDataPath + this.legacyPaths = [ + ["electron", join(userDataPath, CLIENT_STATE_FILENAME)], + ...(legacyTauriDataPath ? [["tauri", join(legacyTauriDataPath, CLIENT_STATE_FILENAME)] as const] : []), + ] + this.removeLegacyState = options?.removeLegacyState ?? ((path) => rmSync(path, { force: true })) this.primary = election try { this.crossHostRegistration = CrossHostRegistration.register( @@ -208,46 +258,118 @@ export class ClientStateManager { } catch (error) { console.warn("[client-state] failed to register cross-host ownership", error) } - if (!this.crossHostRegistration?.isPrimary) { + if (!this.crossHostRegistration) { if (election) removeProcessOwnerLockIfOwned(this.lockPath, this.owner) this.primary = false } if (this.isPrimary) { - const legacyPaths = [ - ["electron", join(userDataPath, CLIENT_STATE_FILENAME)], - ...(legacyTauriDataPath ? [["tauri", join(legacyTauriDataPath, CLIENT_STATE_FILENAME)] as const] : []), - ] as ReadonlyArray - this.migrateLegacyStateIfNeeded(legacyPaths, options?.removeLegacyState) - const futureLegacyBlocked = this.unsupportedFutureEnvelope - const persisted = this.readState() - this.state = futureLegacyBlocked - ? { version: CLIENT_STATE_VERSION, restoreEnabled: false } - : persisted.state - this.persistenceSuppressed = !this.state.restoreEnabled - this.unsupportedFutureEnvelope = futureLegacyBlocked || persisted.unsupportedFutureEnvelope + this.reloadAuthoritativeState() + this.setEffectivePrimary(true, false) } } get isPrimary(): boolean { - if (!this.primary || !this.crossHostRegistration?.isPrimary) return false + let primary = this.primary && Boolean(this.crossHostRegistration?.isPrimary) + if (!primary) { + this.setEffectivePrimary(false) + return false + } if (!this.legacyTauriDataPath) return true try { - return !hasLiveTauriClient(this.legacyTauriDataPath, undefined, undefined, undefined, crossHostParticipants(this.crossHostRegistration.path)) + primary = !hasLiveTauriClient( + this.legacyTauriDataPath, + this.crossHostDependencies?.pidAlive, + this.crossHostDependencies?.processStartIdentity, + undefined, + crossHostParticipants(this.crossHostRegistration!.path), + ) } catch (error) { console.warn("[client-state] failed to recheck legacy Tauri process markers; ownership disabled", error) - return false + primary = false } + if (!primary) this.setEffectivePrimary(false) + return primary } loadClientState(): ClientStateLoadResult { if (!this.isPrimary) { return { isPrimary: false, restoreEnabled: false, snapshot: null } } - return { + const result = { isPrimary: true, restoreEnabled: this.state.restoreEnabled, snapshot: this.state.restoreEnabled ? (this.state.snapshot ?? null) : null, } + this.rendererReconciliationPending = false + return result + } + + onOwnershipChanged(listener: () => void): () => void { + this.ownershipListeners.add(listener) + return () => this.ownershipListeners.delete(listener) + } + + async refreshPrimary(): Promise { + if (this.frozen) return false + if (this.primary && !isProcessOwnerLockOwned(this.lockPath, this.owner)) { + this.primary = false + this.crossHostRegistration?.deferPrimary() + this.setEffectivePrimary(false) + return false + } + if (!this.primary) { + this.primary = electClientStateProcess( + this.userDataPath, + this.owner, + { primaryLockPath: this.lockPath, registrationLockPath: this.registrationLockPath }, + (message, error) => console.warn(`[client-state] ${message}`, error), + this.crossHostDependencies?.pidAlive, + 0, + () => {}, + this.crossHostDependencies?.processStartIdentity, + true, + ) + if (!this.primary) { + const localOwner = readProcessOwnerLock(this.lockPath) + if (localOwner) await this.crossHostRegistration?.participateInRecoveryAsync(localOwner) + return false + } + } + try { + if (this.crossHostRegistration?.isPrimary) { + if (await this.canOwnCrossHostStateAsync()) { + if (this.effectivePrimary) return false + this.reloadAuthoritativeState() + this.rendererReconciliationPending = true + this.setEffectivePrimary(true) + return true + } + this.crossHostRegistration?.deferPrimary() + this.setEffectivePrimary(false) + return false + } + if (!await this.canOwnCrossHostStateAsync()) { + this.crossHostRegistration?.deferPrimary() + this.setEffectivePrimary(false) + return false + } + const registration = this.crossHostRegistration + if (!registration || !await registration.tryAcquireAsync(true)) return false + if (!await this.canOwnCrossHostStateAsync()) { + registration.deferPrimary() + this.setEffectivePrimary(false) + return false + } + this.reloadAuthoritativeState() + this.rendererReconciliationPending = true + this.setEffectivePrimary(true) + return true + } catch (error) { + this.crossHostRegistration?.deferPrimary() + this.setEffectivePrimary(false) + console.warn("[client-state] failed to promote from shared state", error) + return false + } } getWindowState(): NativeWindowState | undefined { @@ -316,7 +438,7 @@ export class ClientStateManager { } clearClientState(rendererToken?: unknown): Promise { - if (!this.isPrimary) { + if (!this.isPrimary || this.rendererReconciliationPending) { return Promise.resolve(false) } if (this.frozen) { @@ -363,12 +485,13 @@ export class ClientStateManager { return this.drainAndReleasePromise } - private readState(): ParsedClientState { + private readState(authoritative = false): ParsedClientState { try { return parseClientState(readFileSync(this.statePath, "utf8")) } catch (error) { if (!hasErrorCode(error, "ENOENT")) { console.warn("[client-state] failed to read state", error) + if (authoritative) throw error } return { state: { version: CLIENT_STATE_VERSION, restoreEnabled: true }, @@ -377,22 +500,22 @@ export class ClientStateManager { } } - private migrateLegacyStateIfNeeded( - paths: ReadonlyArray, - removeLegacyState = (path: string) => rmSync(path, { force: true }), - ): void { + private migrateLegacyStateIfNeeded(): void { try { readFileSync(this.statePath) return } catch (error) { if (!hasErrorCode(error, "ENOENT")) return } - if (paths.some(([, path]) => isFutureLegacyCandidate(path))) { + const legacySnapshots = this.legacyPaths + .map(([host, path]) => snapshotLegacyState(path, host)) + .filter((snapshot): snapshot is LegacyStateSnapshot => Boolean(snapshot)) + if (legacySnapshots.some(isFutureLegacyCandidate)) { this.unsupportedFutureEnvelope = true return } - const winner = paths - .map(([host, path]) => legacyCandidate(path, host)) + const winner = legacySnapshots + .map(legacyCandidate) .filter((candidate): candidate is NonNullable => Boolean(candidate)) .sort((left, right) => Number(left.state.restoreEnabled) - Number(right.state.restoreEnabled) || @@ -410,13 +533,28 @@ export class ClientStateManager { fsyncSync(descriptor) closeSync(descriptor) descriptor = undefined - this.assertReplacementAllowed() + this.assertSharedOwnershipAllowed() renameSync(temporaryPath, this.statePath) - for (const [, path] of paths) { + for (const snapshot of legacySnapshots) { + const quarantinePath = join( + dirname(snapshot.path), + `.${CLIENT_STATE_FILENAME}.${this.owner.pid}.${this.owner.runToken}.${snapshot.host}.migration-quarantine`, + ) try { - removeLegacyState(path) + this.assertSharedOwnershipAllowed() + renameSync(snapshot.path, quarantinePath) + try { + if (!legacySnapshotMatches(snapshot, quarantinePath)) { + throw new Error("legacy state changed before quarantine") + } + this.assertSharedOwnershipAllowed() + this.removeLegacyState(quarantinePath) + } catch (error) { + this.restoreQuarantinedLegacyState(snapshot.path, quarantinePath) + throw error + } } catch (error) { - console.warn(`[client-state] failed to remove migrated legacy state at ${path}`, error) + console.warn(`[client-state] failed to remove migrated legacy state at ${snapshot.path}`, error) } } } finally { @@ -426,7 +564,7 @@ export class ClientStateManager { } private getMutationDisposition(futureEnvelopeResult = true): Promise | undefined { - if (!this.isPrimary) { + if (!this.isPrimary || this.rendererReconciliationPending) { return Promise.resolve(false) } if (this.frozen) { @@ -443,8 +581,13 @@ export class ClientStateManager { skipWhenSuppressed = false, rendererToken?: unknown, ): Promise { + const ownershipEpoch = this.ownershipEpoch const operation = this.writeQueue.catch(() => {}).then(async () => { if (rendererToken !== undefined) this.assertRendererAccessToken(rendererToken) + this.assertReplacementAllowed(ownershipEpoch, rendererToken) + const runtimeSuppression = this.persistenceSuppressed + this.reloadAuthoritativeState() + this.persistenceSuppressed = runtimeSuppression if (skipWhenSuppressed && this.persistenceSuppressed) { return } @@ -454,11 +597,15 @@ export class ClientStateManager { const previousUnsupportedFutureEnvelope = this.unsupportedFutureEnvelope try { mutate(this.state) - await this.writeAtomically(JSON.stringify(this.state), rendererToken) + await this.writeAtomically(JSON.stringify(this.state), ownershipEpoch, rendererToken) } catch (error) { - this.state = previousState - this.persistenceSuppressed = previousPersistenceSuppressed - this.unsupportedFutureEnvelope = previousUnsupportedFutureEnvelope + if (ownershipEpoch === this.ownershipEpoch) { + this.state = previousState + this.persistenceSuppressed = previousPersistenceSuppressed + this.unsupportedFutureEnvelope = previousUnsupportedFutureEnvelope + } else if (this.isPrimary) { + this.reloadAuthoritativeState() + } throw error } }) @@ -466,14 +613,14 @@ export class ClientStateManager { return operation.then(() => true) } - private async writeAtomically(serializedState: string, rendererToken?: unknown): Promise { + private async writeAtomically(serializedState: string, ownershipEpoch: number, rendererToken?: unknown): Promise { const temporaryPath = join( dirname(this.statePath), `.${CLIENT_STATE_FILENAME}.${this.owner.pid}.${this.owner.runToken}.tmp`, ) try { await this.writeState(temporaryPath, serializedState) - this.assertReplacementAllowed(rendererToken) + this.assertReplacementAllowed(ownershipEpoch, rendererToken) await rename(temporaryPath, this.statePath) } catch (error) { await rm(temporaryPath, { force: true }).catch(() => {}) @@ -481,13 +628,76 @@ export class ClientStateManager { } } - private assertReplacementAllowed(rendererToken?: unknown): void { + private assertReplacementAllowed(ownershipEpoch: number, rendererToken?: unknown): void { if (rendererToken !== undefined) this.assertRendererAccessToken(rendererToken) - if (!this.isPrimary || !isProcessOwnerLockOwned(this.lockPath, this.owner)) { + if (!this.isPrimary || this.ownershipEpoch !== ownershipEpoch || !isProcessOwnerLockOwned(this.lockPath, this.owner)) { throw new Error("Client state ownership changed before atomic replacement") } } + private restoreQuarantinedLegacyState(path: string, quarantinePath: string): void { + try { + linkSync(quarantinePath, path) + rmSync(quarantinePath, { force: true }) + } catch (error) { + if (!hasErrorCode(error, "EEXIST") && !hasErrorCode(error, "ENOENT")) { + console.warn(`[client-state] failed to restore quarantined legacy state at ${path}`, error) + } + } + } + + private canOwnCrossHostState(): boolean { + if (!this.primary || !isProcessOwnerLockOwned(this.lockPath, this.owner)) return false + if (!this.legacyTauriDataPath) return true + return !hasLiveTauriClient( + this.legacyTauriDataPath, + this.crossHostDependencies?.pidAlive, + this.crossHostDependencies?.processStartIdentity, + undefined, + crossHostParticipants(this.crossHostRegistration?.path ?? ""), + ) + } + + private async canOwnCrossHostStateAsync(): Promise { + if (!this.primary || !isProcessOwnerLockOwned(this.lockPath, this.owner)) return false + if (!this.legacyTauriDataPath) return true + return !await hasLiveTauriClientAsync( + this.legacyTauriDataPath, + this.crossHostDependencies?.pidAlive, + this.crossHostDependencies?.processStartIdentityAsync ?? this.crossHostDependencies?.processStartIdentity, + undefined, + crossHostParticipants(this.crossHostRegistration?.path ?? ""), + ) + } + + private reloadAuthoritativeState(): void { + this.unsupportedFutureEnvelope = false + this.migrateLegacyStateIfNeeded() + const futureLegacyBlocked = this.unsupportedFutureEnvelope + const persisted = this.readState(true) + this.state = futureLegacyBlocked + ? { version: CLIENT_STATE_VERSION, restoreEnabled: false } + : persisted.state + this.persistenceSuppressed = !this.state.restoreEnabled + this.unsupportedFutureEnvelope = futureLegacyBlocked || persisted.unsupportedFutureEnvelope + } + + private assertSharedOwnershipAllowed(): void { + if (!this.crossHostRegistration?.isPrimary || !this.canOwnCrossHostState()) { + throw new Error("Client state ownership changed before atomic replacement") + } + } + + private setEffectivePrimary(primary: boolean, notify = true): void { + if (this.effectivePrimary === primary) return + this.effectivePrimary = primary + this.ownershipEpoch += 1 + if (notify) { + this.rendererReconciliationPending = true + for (const listener of this.ownershipListeners) listener() + } + } + private releaseOwnedProcessFiles(): void { const releases: Array<[string, () => void]> = [ ["remove running marker", () => { removeRunningMarkerIfOwned(getRunningMarkerPath(this.userDataPath, this.owner), this.owner) }], diff --git a/packages/electron-app/electron/main/main.ts b/packages/electron-app/electron/main/main.ts index f90fb3611..e31595feb 100644 --- a/packages/electron-app/electron/main/main.ts +++ b/packages/electron-app/electron/main/main.ts @@ -98,12 +98,36 @@ cleanupPackagedChromiumStorage() const clientStateManager = new ClientStateManager(app.getPath("userData")) const cliManager = new CliProcessManager() let mainWindow: BrowserWindow | null = null +let mainWindowStateTracker: WindowStateTracker | null = null let currentCliUrl: string | null = null let pendingCliUrl: string | null = null let pendingBootstrapToken: string | null = null let showingLoadingScreen = false let preloadingView: BrowserView | null = null let mainNavigationController: ClientStateNavigationController | null = null +clientStateManager.onOwnershipChanged(() => { + if (mainWindow && !mainWindow.isDestroyed()) { + const windowState = clientStateManager.getWindowState() + const bounds = windowState + ? clampWindowBounds(windowState.bounds, screen.getAllDisplays().map((display) => display.workArea)) + : undefined + mainWindowStateTracker?.applyAuthoritativeState(windowState, bounds) + mainWindow.webContents.send("client-state:ownership-changed") + } +}) +const pollClientStateOwnership = () => { + const timer = setTimeout(async () => { + try { + await clientStateManager.refreshPrimary() + } catch (error) { + console.warn("[client-state] ownership poll failed", error) + } finally { + pollClientStateOwnership() + } + }, 250) + timer.unref() +} +pollClientStateOwnership() const remoteWindowOrigins = new Map>() const insecureWindowOrigins = new Map>() const clientStateLifecycle = new ClientStateLifecycle({ @@ -413,10 +437,10 @@ function createWindow() { ) mainNavigationController = navigationController - let windowStateTracker: WindowStateTracker | null = null + const windowStateTracker = new WindowStateTracker(window, clientStateManager, savedWindowState) + mainWindowStateTracker = windowStateTracker if (clientStateManager.isPrimary) { restoreWindowState(window, savedWindowState, restoredBounds) - windowStateTracker = new WindowStateTracker(window, clientStateManager, savedWindowState) } installWindowZoomInput(window, (level) => { if (windowStateTracker) windowStateTracker.setZoomLevel(level) @@ -455,6 +479,7 @@ function createWindow() { clearWindowAllowedOrigin(window) clearWindowInsecureOrigin(window) mainWindow = null + if (mainWindowStateTracker === windowStateTracker) mainWindowStateTracker = null if (mainNavigationController === navigationController) mainNavigationController = null currentCliUrl = null pendingCliUrl = null diff --git a/packages/electron-app/electron/main/window-state.test.ts b/packages/electron-app/electron/main/window-state.test.ts index fae89974e..514221e76 100644 --- a/packages/electron-app/electron/main/window-state.test.ts +++ b/packages/electron-app/electron/main/window-state.test.ts @@ -26,13 +26,51 @@ test("restores shared outer position and content size", () => { const window = { setPosition: (x: number, y: number) => calls.push(["position", x, y]), setContentSize: (width: number, height: number) => calls.push(["content", width, height]), + unmaximize: () => calls.push(["unmaximize"]), maximize: () => undefined, - setFullScreen: () => undefined, + setFullScreen: (enabled: boolean) => calls.push(["fullscreen", enabled]), webContents: { setZoomFactor: () => undefined }, } as unknown as BrowserWindow const bounds = { x: 10, y: 20, width: 1200, height: 800 } restoreWindowState(window, { bounds, maximized: false, fullscreen: false, zoomFactor: 1 }, bounds) - assert.deepEqual(calls, [["position", 10, 20], ["content", 1200, 800]]) + assert.deepEqual(calls, [["fullscreen", false], ["unmaximize"], ["position", 10, 20], ["content", 1200, 800]]) +}) + +test("late promotion applies authoritative window state and tracker baseline", async () => { + const calls: unknown[] = [] + let maximized = false + const window = { + isDestroyed: () => false, + on: () => undefined, + getPosition: () => [1, 2], + getContentSize: () => [900, 700], + isMaximized: () => maximized, + isFullScreen: () => false, + setPosition: (x: number, y: number) => calls.push(["position", x, y]), + setContentSize: (width: number, height: number) => calls.push(["content", width, height]), + unmaximize: () => calls.push(["unmaximize"]), + maximize: () => { maximized = true; calls.push(["maximize"]) }, + setFullScreen: (enabled: boolean) => calls.push(["fullscreen", enabled]), + webContents: { + isDestroyed: () => false, + on: () => undefined, + setZoomFactor: (factor: number) => calls.push(["zoom", factor]), + getZoomFactor: () => 1.5, + }, + } as unknown as BrowserWindow + const saved: any[] = [] + const manager = { + saveWindowState: async (state: unknown) => { saved.push(state); return true }, + flush: async () => undefined, + } as unknown as ClientStateManager + const tracker = new WindowStateTracker(window, manager) + const bounds = { x: 10, y: 20, width: 1200, height: 800 } + tracker.applyAuthoritativeState({ bounds, maximized: true, fullscreen: false, zoomFactor: 1.5 }, bounds) + await tracker.flush() + + assert.deepEqual(calls.slice(0, 5), [["fullscreen", false], ["position", 10, 20], ["content", 1200, 800], ["zoom", 1.5], ["maximize"]]) + assert.deepEqual(saved[0].bounds, bounds) + assert.equal(saved[0].zoomFactor, 1.5) }) test("flush captures the current native zoom", async () => { diff --git a/packages/electron-app/electron/main/window-state.ts b/packages/electron-app/electron/main/window-state.ts index b595b495f..a60e24183 100644 --- a/packages/electron-app/electron/main/window-state.ts +++ b/packages/electron-app/electron/main/window-state.ts @@ -117,6 +117,8 @@ export function restoreWindowState(window: BrowserWindow, state: NativeWindowSta return } + if (!state.fullscreen) window.setFullScreen(false) + if (!state.maximized) window.unmaximize() if (bounds) { window.setPosition(bounds.x, bounds.y) window.setContentSize(bounds.width, bounds.height) @@ -206,6 +208,14 @@ export class WindowStateTracker { this.scheduleSave() } + applyAuthoritativeState(state: NativeWindowState | undefined, bounds: WindowBounds | undefined): void { + if (!state || this.window.isDestroyed()) return + this.clearTimer() + this.normalBounds = bounds ?? state.bounds + this.desiredZoomFactor = normalizeZoomFactor(state.zoomFactor) + restoreWindowState(this.window, state, bounds) + } + private scheduleSave() { this.clearTimer() this.saveTimer = setTimeout(() => { diff --git a/packages/electron-app/electron/preload/index.cjs b/packages/electron-app/electron/preload/index.cjs index 0a35f6690..f0349331b 100644 --- a/packages/electron-app/electron/preload/index.cjs +++ b/packages/electron-app/electron/preload/index.cjs @@ -43,6 +43,11 @@ const localElectronAPI = { setClientStateRestoreEnabled: (token, enabled) => ipcRenderer.invoke("client-state:setRestoreEnabled", token, Boolean(enabled)), clearClientState: (token) => ipcRenderer.invoke("client-state:clear", token), + onClientStateOwnershipChange: (callback) => { + const listener = () => callback() + ipcRenderer.on("client-state:ownership-changed", listener) + return () => ipcRenderer.removeListener("client-state:ownership-changed", listener) + }, } const remoteElectronAPI = { diff --git a/packages/opencode-plugin/plugin/codenomad.ts b/packages/opencode-plugin/plugin/codenomad.ts index 61d1827f0..cee264840 100644 --- a/packages/opencode-plugin/plugin/codenomad.ts +++ b/packages/opencode-plugin/plugin/codenomad.ts @@ -1,17 +1,19 @@ import type { PluginInput } from "@opencode-ai/plugin" import { createCodeNomadClient, getCodeNomadConfig } from "./lib/client.js" import { createBackgroundProcessTools } from "./lib/background-process.js" +import { createWorkflowTools } from "./lib/workflows.js" let voiceModeEnabled = false export async function CodeNomadPlugin(input: PluginInput): Promise<{ - tool: ReturnType + tool: ReturnType & ReturnType "chat.message": CodeNomadChatMessageHook event: CodeNomadEventHook }> { const config = getCodeNomadConfig() const client = createCodeNomadClient(config) const backgroundProcessTools = createBackgroundProcessTools(config, { baseDir: input.directory }) + const workflowTools = createWorkflowTools(config) await client.startEvents((event) => { if (event.type === "codenomad.ping") { @@ -33,6 +35,7 @@ export async function CodeNomadPlugin(input: PluginInput): Promise<{ return { tool: { ...backgroundProcessTools, + ...workflowTools, }, async "chat.message"(_input: { sessionID: string }, output: { message: { system?: string } }) { if (!voiceModeEnabled) { diff --git a/packages/opencode-plugin/plugin/lib/request.test.ts b/packages/opencode-plugin/plugin/lib/request.test.ts new file mode 100644 index 000000000..f37395443 --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/request.test.ts @@ -0,0 +1,116 @@ +import assert from "node:assert/strict" +import { createServer } from "node:http" +import https from "node:https" +import { test } from "node:test" + +import { createCodeNomadRequester } from "./request" + +test("plugin HTTPS requests retain native certificate verification", async () => { + const originalRequest = https.request + let requestOptions: https.RequestOptions | undefined + https.request = ((options: https.RequestOptions) => { + requestOptions = options + throw new Error("request intercepted") + }) as typeof https.request + + try { + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: "https://127.0.0.1:443", + callbackToken: "workspace-callback", + }) + await assert.rejects(requester.requestVoid("/event"), /request intercepted/) + assert.equal(requestOptions?.rejectUnauthorized, undefined) + assert.equal(requestOptions?.agent, undefined) + } finally { + https.request = originalRequest + } +}) + +test("plugin requests use the distinct callback capability", async () => { + let authorization: string | undefined + const server = createServer((request, response) => { + authorization = request.headers.authorization + response.writeHead(200, { Connection: "close" }).end() + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "workspace-callback", + }) + + await requester.requestVoid("/event", { method: "POST" }) + + assert.equal(authorization, "Bearer workspace-callback") + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) + +test("pre-aborted plugin requests reject without an unhandled request error", async () => { + let requests = 0 + const server = createServer((_request, response) => { + requests += 1 + response.writeHead(200, { Connection: "close" }).end() + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "workspace-callback", + }) + const controller = new AbortController() + controller.abort() + + await assert.rejects(requester.requestVoid("/event", { signal: controller.signal }), { name: "AbortError" }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(requests, 0) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) + +test("plugin responses use null bodies when HTTP forbids response content", async () => { + const server = createServer((request, response) => { + const status = Number(new URL(request.url ?? "/", "http://localhost").pathname.slice(1)) || 200 + response.writeHead(status, { Connection: "close" }).end("ignored") + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const requester = createCodeNomadRequester({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "workspace-callback", + }) + + for (const status of [204, 205, 304]) { + const response = await requester.fetch(`http://127.0.0.1:${address.port}/${status}`) + assert.equal(response.status, status) + assert.equal(response.body, null) + } + assert.equal(await requester.requestJson(`http://127.0.0.1:${address.port}/205`), undefined) + const head = await requester.fetch(`http://127.0.0.1:${address.port}/200`, { method: "HEAD" }) + assert.equal(head.body, null) + assert.equal(await requester.requestJson(`http://127.0.0.1:${address.port}/200`, { method: "HEAD" }), undefined) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) diff --git a/packages/opencode-plugin/plugin/lib/request.ts b/packages/opencode-plugin/plugin/lib/request.ts index 5025a5013..8b48877dd 100644 --- a/packages/opencode-plugin/plugin/lib/request.ts +++ b/packages/opencode-plugin/plugin/lib/request.ts @@ -10,12 +10,14 @@ export type PluginEvent = { export type CodeNomadConfig = { instanceId: string baseUrl: string + callbackToken: string } export function getCodeNomadConfig(): CodeNomadConfig { return { instanceId: requireEnv("CODENOMAD_INSTANCE_ID"), baseUrl: requireEnv("CODENOMAD_BASE_URL"), + callbackToken: requireEnv("CODENOMAD_CALLBACK_TOKEN"), } } @@ -23,7 +25,7 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { const rawBaseUrl = (config.baseUrl ?? "").trim() const baseUrl = rawBaseUrl.replace(/\/+$/, "") const pluginBase = `${baseUrl}/workspaces/${encodeURIComponent(config.instanceId)}/plugin` - const authorization = buildInstanceAuthorizationHeader() + const authorization = `Bearer ${config.callbackToken}` const buildUrl = (path: string) => { if (path.startsWith("http://") || path.startsWith("https://")) { @@ -47,10 +49,7 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { const hasBody = init?.body !== undefined const headers = buildHeaders(init?.headers, hasBody) - // The CodeNomad plugin only talks to the local CodeNomad server. - // Use a single request implementation that tolerates custom/self-signed certs - // without disabling TLS verification for the whole Node process. - return nodeFetch(url, { ...init, headers }, { rejectUnauthorized: false }) + return nodeFetch(url, { ...init, headers }) } const requestJson = async (path: string, init?: RequestInit): Promise => { @@ -60,7 +59,7 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { throw new Error(message || `Request failed with ${response.status}`) } - if (response.status === 204) { + if ((init?.method ?? "GET").toUpperCase() === "HEAD" || response.status === 204 || response.status === 205) { return undefined as T } @@ -95,7 +94,6 @@ export function createCodeNomadRequester(config: CodeNomadConfig) { async function nodeFetch( url: string, init: RequestInit & { headers?: Record }, - tls: { rejectUnauthorized: boolean }, ): Promise { const parsed = new URL(url) const isHttps = parsed.protocol === "https:" @@ -114,9 +112,9 @@ async function nodeFetch( path: `${parsed.pathname}${parsed.search}`, method, headers, - ...(isHttps ? { rejectUnauthorized: tls.rejectUnauthorized } : {}), }, (res) => { + const status = res.statusCode ?? 0 const responseHeaders = new Headers() for (const [key, value] of Object.entries(res.headers)) { if (value === undefined) continue @@ -127,9 +125,10 @@ async function nodeFetch( } } - // Convert Node stream -> Web ReadableStream for Response. - const webBody = Readable.toWeb(res) as unknown as ReadableStream - resolve(new Response(webBody, { status: res.statusCode ?? 0, headers: responseHeaders })) + const bodyForbidden = method === "HEAD" || status === 204 || status === 205 || status === 304 + if (bodyForbidden) res.resume() + const webBody = bodyForbidden ? null : Readable.toWeb(res) as unknown as ReadableStream + resolve(new Response(webBody, { status, headers: responseHeaders })) }, ) @@ -141,6 +140,8 @@ async function nodeFetch( reject(err) } + req.once("error", reject) + if (signal) { if (signal.aborted) { abort() @@ -150,8 +151,6 @@ async function nodeFetch( req.once("close", () => signal.removeEventListener("abort", abort)) } - req.once("error", reject) - if (body === undefined || body === null) { req.end() return @@ -185,13 +184,6 @@ function requireEnv(key: string): string { return value } -function buildInstanceAuthorizationHeader(): string { - const username = requireEnv("OPENCODE_SERVER_USERNAME") - const password = requireEnv("OPENCODE_SERVER_PASSWORD") - const token = Buffer.from(`${username}:${password}`, "utf8").toString("base64") - return `Basic ${token}` -} - function normalizeHeaders(headers: HeadersInit | undefined): Record { const output: Record = {} if (!headers) return output diff --git a/packages/opencode-plugin/plugin/lib/workflows.test.ts b/packages/opencode-plugin/plugin/lib/workflows.test.ts new file mode 100644 index 000000000..75d43cbc1 --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/workflows.test.ts @@ -0,0 +1,318 @@ +import assert from "node:assert/strict" +import { createServer } from "node:http" +import test from "node:test" +import { + createWorkflowTools, + describeWorkflowDefinition, + describeWorkflowDefinitions, + describeWorkflowDetails, + parseWorkflowInputs, +} from "./workflows.js" + +const run = { + id: "run", + objective: "Ship it", + status: "running" as const, + steps: [{ id: "build", title: "Build", status: "pending" }], +} + +test("workflow review messaging handles final and truncated gates", () => { + const waiting = { + ...run, + status: "waiting_for_review" as const, + pendingReviewStepId: "build", + steps: [{ + id: "build", + title: "Build", + status: "completed", + sessionId: "session-1", + output: "partial", + outputTruncated: true, + }], + } + const details = describeWorkflowDetails(waiting) + assert.match(details, /continue or complete/) + assert.match(details, /truncated/) + assert.match(details, /session-1/) +}) + +test("dynamic workflow details include execution progress, usage, statuses, and gate guidance", () => { + const dynamic = { + id: "dynamic-run", + objective: "Deploy", + status: "waiting_for_input" as const, + definitionId: "deploy", + definitionRevision: 3, + steps: [], + executionNodes: [ + { instanceKey: "plan", status: "completed", sessionIds: ["session-1"] }, + { instanceKey: "environment", status: "waiting" }, + ], + pendingGate: { + executionNodeId: "gate-execution-id", + gate: "input" as const, + prompt: "Choose an environment", + inputSchema: { type: "string", enum: ["staging", "production"] }, + }, + usage: { + cost: 0.25, + tokens: 120, + inputTokens: 70, + outputTokens: 40, + reasoningTokens: 10, + cacheReadTokens: 5, + cacheWriteTokens: 0, + }, + } + const message = describeWorkflowDetails(dynamic) + assert.match(message, /Status: waiting_for_input/) + assert.match(message, /Execution nodes: 2 total \(completed: 1, waiting: 1\)/) + assert.match(message, /Usage: 120 tokens/) + assert.match(message, /Choose an environment/) + assert.match(message, /Expected input schema/) + assert.match(message, /human in the CodeNomad UI/) + assert.match(message, /cannot answer this gate/) + assert.match(message, /session-1/) + + const approval = describeWorkflowDetails({ + ...dynamic, + status: "waiting_for_review", + pendingGate: { ...dynamic.pendingGate, gate: "approval" }, + }) + assert.match(approval, /cannot approve this gate/) + + for (const status of ["pausing", "paused", "recovery_required"] as const) { + assert.match(describeWorkflowDetails({ ...dynamic, status, pendingGate: undefined }), new RegExp(`Status: ${status}`)) + assert.match(describeWorkflowDetails({ ...dynamic, status, pendingGate: undefined }), /CodeNomad UI/) + } +}) + +test("saved workflow definition messages expose current revision and canonical definition", () => { + const definition = { + id: "deploy", + revision: 3, + definition: { name: "Deploy", description: "Deploy safely" }, + canonical: '{"version":1,"id":"deploy"}', + } + assert.match(describeWorkflowDefinitions([definition]), /deploy \| revision 3 \| Deploy \| Deploy safely/) + assert.match(describeWorkflowDefinition(definition), /Canonical definition:\n\{"version":1/) + assert.equal(describeWorkflowDefinitions([]), "No saved CodeNomad workflow definitions found.") +}) + +test("saved workflow inputs require a JSON object", () => { + assert.deepEqual(parseWorkflowInputs('{"environment":"staging"}'), { environment: "staging" }) + assert.equal(parseWorkflowInputs(), undefined) + assert.throws(() => parseWorkflowInputs("not-json"), /valid JSON/) + assert.throws(() => parseWorkflowInputs("[]"), /JSON object/) + assert.throws(() => parseWorkflowInputs("null"), /JSON object/) + let nested: unknown = true + for (let depth = 0; depth < 21; depth++) nested = { nested } + assert.throws(() => parseWorkflowInputs(JSON.stringify(nested)), /deeply nested/) + assert.throws(() => parseWorkflowInputs(JSON.stringify({ values: Array(50_001).fill(null) })), /too many values/) + assert.throws(() => parseWorkflowInputs(JSON.stringify({ value: "é".repeat(128_001) })), /too large/) +}) + +test("saved definition tools read and start the current revision without claiming session ancestry", async () => { + const calls: Array<{ path: string; init?: RequestInit }> = [] + const definition = { + id: "deploy_flow", + revision: 3, + definition: { name: "Deploy" }, + canonical: '{"version":1}', + } + const requester = { + async requestJson(path: string, init?: RequestInit): Promise { + calls.push({ path, init }) + if (path.endsWith("/start")) { + const runId = JSON.parse(String(init?.body)).runId + return { + id: runId, + objective: "Ship it", + status: "running", + definitionId: "deploy_flow", + definitionRevision: 3, + steps: [], + executionNodes: [], + } as T + } + if (path === "/workflow-definitions" && !init) return { definitions: [definition] } as T + return definition as T + }, + } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + assert.equal("start_codenomad_workflow" in tools, false) + + await tools.list_codenomad_workflow_definitions.execute({}, {} as never) + await tools.get_codenomad_workflow_definition.execute({ definition_id: "deploy_flow" }, {} as never) + const abort = new AbortController().signal + const started = await tools.start_codenomad_workflow_definition.execute({ + definition_id: "deploy_flow", + objective: "Ship it", + inputs_json: '{"environment":"production"}', + }, { sessionID: "session-1", abort } as never) + + assert.deepEqual(calls.map((call) => call.path), [ + "/workflow-definitions", + "/workflow-definitions/deploy_flow", + "/workflow-definitions/deploy_flow/start", + ]) + assert.equal(calls[2]?.init?.method, "POST") + assert.equal(calls[2]?.init?.signal instanceof AbortSignal, true) + const body = JSON.parse(String(calls[2]?.init?.body)) + assert.match(body.runId, /^[0-9a-f-]{36}$/) + assert.deepEqual(body, { + runId: body.runId, + objective: "Ship it", + inputs: { environment: "production" }, + }) + assert.match(started, /current saved definition revision/) + assert.doesNotMatch(String(calls[2]?.init?.body), /definitionRevision|initiatorSessionId/) +}) + +test("saved definition starts finish acceptance and cancel when aborted in flight", async () => { + let accept!: (value: typeof run) => void + let runId = "" + const calls: Array<{ path: string; init?: RequestInit }> = [] + const requester = { + async requestJson(path: string, init?: RequestInit): Promise { + calls.push({ path, init }) + if (path.endsWith("/start")) { + runId = JSON.parse(String(init?.body)).runId + return await new Promise((resolve) => { accept = resolve as typeof accept }) + } + return { ...run, status: "cancelled" } as T + }, + } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + const controller = new AbortController() + const started = tools.start_codenomad_workflow_definition.execute({ definition_id: "deploy" }, { abort: controller.signal } as never) + controller.abort() + accept(run) + + await assert.rejects(started, { name: "AbortError" }) + assert.deepEqual(calls.map(({ path }) => path), [ + "/workflow-definitions/deploy/start", + `/workflow-runs/${runId}/cancel`, + ]) + assert.equal(calls[0]?.init?.signal instanceof AbortSignal, true) + assert.notEqual(calls[1]?.init?.signal, controller.signal) + assert.equal(calls[1]?.init?.signal instanceof AbortSignal, true) +}) + +test("saved definition starts expose the run ID when compensating cancellation fails", async () => { + let runId = "" + const requester = { + async requestJson(path: string, init?: RequestInit): Promise { + if (path.endsWith("/start")) { + runId = JSON.parse(String(init?.body)).runId + await new Promise((resolve) => setImmediate(resolve)) + return run as T + } + throw new Error("cancel unavailable") + }, + } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + const controller = new AbortController() + const started = tools.start_codenomad_workflow_definition.execute({ definition_id: "deploy" }, { abort: controller.signal } as never) + controller.abort() + + await assert.rejects(started, new RegExp(`Workflow ${runId} may have started but cancellation failed: cancel unavailable`)) +}) + +test("saved definition starts cancel malformed successful responses using the requested run ID", async () => { + for (const response of [ + () => ({}), + (runId: string) => ({ id: runId, objective: "Ship it", status: "running", steps: {} }), + ]) { + let runId = "" + const calls: string[] = [] + const requester = { + async requestJson(path: string, init?: RequestInit): Promise { + calls.push(path) + if (path.endsWith("/start")) { + runId = JSON.parse(String(init?.body)).runId + return response(runId) as T + } + return { ...run, id: runId, status: "cancelled" } as T + }, + } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + + await assert.rejects( + tools.start_codenomad_workflow_definition.execute({ definition_id: "deploy" }, { abort: new AbortController().signal } as never), + new RegExp(`Workflow ${runId || "[0-9a-f-]{36}"} start returned a malformed run`), + ) + assert.match(runId, /^[0-9a-f-]{36}$/) + assert.deepEqual(calls, ["/workflow-definitions/deploy/start", `/workflow-runs/${runId}/cancel`]) + } +}) + +test("saved definition starts cancel their known run after an accepted response resets", async () => { + let acceptedRunId = "" + let cancelledRunId = "" + const server = createServer((request, response) => { + let body = "" + request.setEncoding("utf8") + request.on("data", (chunk) => { body += chunk }) + request.on("end", () => { + if (request.url?.endsWith("/start")) { + acceptedRunId = JSON.parse(body).runId + response.writeHead(200, { "Content-Type": "application/json" }) + response.write('{"id":"truncated') + response.destroy() + return + } + cancelledRunId = request.url?.split("/").at(-2) ?? "" + response.writeHead(200, { "Content-Type": "application/json", Connection: "close" }) + response.end(JSON.stringify({ ...run, id: cancelledRunId, status: "cancelled" })) + }) + }) + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)) + try { + const address = server.address() + assert.ok(address && typeof address === "object") + const tools = createWorkflowTools({ + instanceId: "workspace", + baseUrl: `http://127.0.0.1:${address.port}`, + callbackToken: "callback", + }) + + await assert.rejects(tools.start_codenomad_workflow_definition.execute( + { definition_id: "deploy" }, + { abort: new AbortController().signal } as never, + )) + assert.match(acceptedRunId, /^[0-9a-f-]{36}$/) + assert.equal(cancelledRunId, acceptedRunId) + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()) + server.closeAllConnections() + }) + } +}) + +test("pre-aborted saved definition starts do not reach the server", async () => { + let calls = 0 + const requester = { async requestJson(): Promise { calls += 1; return run as T } } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + const controller = new AbortController() + controller.abort() + + await assert.rejects( + tools.start_codenomad_workflow_definition.execute({ definition_id: "deploy" }, { abort: controller.signal } as never), + { name: "AbortError" }, + ) + assert.equal(calls, 0) +}) + +test("workflow cancellation forwards the tool abort signal", async () => { + let init: RequestInit | undefined + const requester = { async requestJson(_path: string, requestInit?: RequestInit): Promise { + init = requestInit + return run as T + } } + const tools = createWorkflowTools({ instanceId: "workspace", baseUrl: "http://localhost", callbackToken: "callback" }, requester) + const abort = new AbortController().signal + await tools.cancel_codenomad_workflow.execute({ run_id: "run" }, { abort } as never) + assert.equal(init?.signal, abort) +}) diff --git a/packages/opencode-plugin/plugin/lib/workflows.ts b/packages/opencode-plugin/plugin/lib/workflows.ts new file mode 100644 index 000000000..f4bd7c143 --- /dev/null +++ b/packages/opencode-plugin/plugin/lib/workflows.ts @@ -0,0 +1,371 @@ +import { tool } from "@opencode-ai/plugin/tool" +import { randomUUID } from "node:crypto" +import { createCodeNomadRequester, type CodeNomadConfig } from "./request.js" + +type WorkflowStatus = + | "running" + | "pausing" + | "paused" + | "waiting_for_review" + | "waiting_for_input" + | "completed" + | "failed" + | "cancelled" + | "interrupted" + | "recovery_required" + +type WorkflowUsage = { + cost: number + tokens: number + inputTokens: number + outputTokens: number + reasoningTokens: number + cacheReadTokens: number + cacheWriteTokens: number +} + +type WorkflowRun = { + id: string + objective: string + status: WorkflowStatus + error?: string + pendingReviewStepId?: string + definitionId?: string + definitionRevision?: number + steps: Array<{ + id: string + title: string + status: string + sessionId?: string + output?: unknown + outputTruncated?: boolean + }> + executionNodes?: Array<{ + instanceKey: string + status: string + sessionIds?: string[] + error?: string + outputTruncated?: boolean + usage?: WorkflowUsage + }> + pendingGate?: { + executionNodeId: string + gate: "approval" | "input" + prompt: string + inputSchema?: Record + } + usage?: WorkflowUsage +} + +type WorkflowDefinitionRecord = { + id: string + revision: number + definition: { + name: string + description?: string + } + canonical: string +} + +const JSON_VALUE_BYTES_LIMIT = 256_000 +const JSON_VALUE_DEPTH_LIMIT = 20 +const JSON_VALUE_COUNT_LIMIT = 50_000 +const START_REQUEST_TIMEOUT_MS = 30_000 +const START_CANCEL_TIMEOUT_MS = 5_000 +const WORKFLOW_STATUSES = new Set([ + "running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "completed", "failed", "cancelled", + "interrupted", "recovery_required", +]) + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value) +} + +function hasOptionalType(candidate: Record, key: string, type: "string" | "number" | "boolean"): boolean { + return candidate[key] === undefined || typeof candidate[key] === type +} + +function isWorkflowUsage(value: unknown): boolean { + if (!isRecord(value)) return false + return ["cost", "tokens", "inputTokens", "outputTokens", "reasoningTokens", "cacheReadTokens", "cacheWriteTokens"] + .every((key) => typeof value[key] === "number" && Number.isFinite(value[key])) +} + +function validateStartedWorkflowRun(value: unknown, runId: string): WorkflowRun { + if (!isRecord(value)) throw new Error(`Workflow ${runId} start returned a malformed run`) + const candidate = value + if ( + candidate.id !== runId || + typeof candidate.objective !== "string" || !WORKFLOW_STATUSES.has(candidate.status as WorkflowStatus) || + !hasOptionalType(candidate, "error", "string") || !hasOptionalType(candidate, "pendingReviewStepId", "string") || + !hasOptionalType(candidate, "definitionId", "string") || !hasOptionalType(candidate, "definitionRevision", "number") || + (candidate.usage !== undefined && !isWorkflowUsage(candidate.usage)) || + !Array.isArray(candidate.steps) || candidate.steps.some((step) => !isRecord(step) || + typeof step.id !== "string" || typeof step.title !== "string" || typeof step.status !== "string" || + !hasOptionalType(step, "sessionId", "string") || !hasOptionalType(step, "outputTruncated", "boolean")) || + (candidate.executionNodes !== undefined && (!Array.isArray(candidate.executionNodes) || candidate.executionNodes.some((node) => + !isRecord(node) || typeof node.instanceKey !== "string" || typeof node.status !== "string" || + !hasOptionalType(node, "error", "string") || !hasOptionalType(node, "outputTruncated", "boolean") || + (node.sessionIds !== undefined && (!Array.isArray(node.sessionIds) || node.sessionIds.some((id) => typeof id !== "string"))) || + (node.usage !== undefined && !isWorkflowUsage(node.usage))))) || + (candidate.pendingGate !== undefined && (!isRecord(candidate.pendingGate) || + typeof candidate.pendingGate.executionNodeId !== "string" || + (candidate.pendingGate.gate !== "approval" && candidate.pendingGate.gate !== "input") || + typeof candidate.pendingGate.prompt !== "string" || + (candidate.pendingGate.inputSchema !== undefined && !isRecord(candidate.pendingGate.inputSchema)))) + ) { + throw new Error(`Workflow ${runId} start returned a malformed run`) + } + return candidate as unknown as WorkflowRun +} + +function summarize(run: WorkflowRun) { + const dynamic = Boolean(run.definitionId || run.executionNodes) + const progress = dynamic + ? summarizeExecution(run) + : run.steps.map((step) => `${step.title}: ${step.status}${step.sessionId ? ` (${step.sessionId})` : ""}`).join("\n") + const guidance = statusGuidance(run.status) + return [ + `Workflow ${run.id}`, + `Status: ${run.status}`, + progress, + guidance, + dynamic ? describePendingGate(run) : "", + run.error ? `Error: ${run.error}` : "", + ].filter(Boolean).join("\n") +} + +function details(run: WorkflowRun) { + if (run.definitionId || run.executionNodes) { + const nodes = (run.executionNodes ?? []).map((node) => { + const sessions = node.sessionIds?.length ? ` (${node.sessionIds.join(", ")})` : "" + const usage = node.usage ? ` | ${node.usage.tokens} tokens, cost ${node.usage.cost}` : "" + const error = node.error ? ` | error: ${node.error}` : "" + const truncated = node.outputTruncated ? " | output truncated; inspect the session in CodeNomad" : "" + return `${node.instanceKey}: ${node.status}${sessions}${usage}${error}${truncated}` + }) + return [summarize(run), nodes.length ? `Execution details:\n${nodes.join("\n")}` : ""].filter(Boolean).join("\n") + } + const reviewed = run.steps.find((step) => step.id === run.pendingReviewStepId) + const output = reviewed?.output === undefined ? "" : `\nPending review:\n${JSON.stringify(reviewed.output, null, 2)}` + const truncated = reviewed?.outputTruncated + ? `\nThis output is truncated. Review the full generated session${reviewed.sessionId ? ` ${reviewed.sessionId}` : ""} before approval.` + : "" + return `${summarize(run)}${output}${truncated}` +} + +export const describeWorkflowDetails = details + +export function parseWorkflowInputs(value?: string): Record | undefined { + if (value === undefined) return undefined + if (Buffer.byteLength(value, "utf8") > JSON_VALUE_BYTES_LIMIT) { + throw new Error("Workflow inputs are too large.") + } + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + throw new Error("Workflow inputs must be valid JSON.") + } + if (parsed === null || Array.isArray(parsed) || typeof parsed !== "object") { + throw new Error("Workflow inputs must be a JSON object.") + } + const issue = inspectJsonValue(parsed) + if (issue) throw new Error(`Workflow inputs ${issue}.`) + return parsed as Record +} + +function inspectJsonValue(input: unknown): string | undefined { + const pending: Array<{ value: unknown; depth: number }> = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let count = 0 + + while (pending.length) { + const { value, depth } = pending.pop()! + if (++count > JSON_VALUE_COUNT_LIMIT) return "contain too many values" + if (depth > JSON_VALUE_DEPTH_LIMIT) return "are too deeply nested" + if (value === null || typeof value === "string" || typeof value === "boolean") continue + if (typeof value === "number" && Number.isFinite(value)) continue + if (!value || typeof value !== "object") return "must contain only JSON values" + if (seen.has(value)) return "must not contain cycles or aliases" + seen.add(value) + + if (Array.isArray(value)) { + const keys = Object.keys(value) + if (keys.length !== value.length || keys.some((key, index) => key !== String(index))) { + return "must contain only plain JSON arrays" + } + for (let index = value.length - 1; index >= 0; index--) { + pending.push({ value: value[index], depth: depth + 1 }) + } + continue + } + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return "must contain only plain JSON objects" + for (const child of Object.values(value)) pending.push({ value: child, depth: depth + 1 }) + } +} + +function summarizeExecution(run: WorkflowRun) { + const nodes = run.executionNodes ?? [] + const counts = new Map() + for (const node of nodes) counts.set(node.status, (counts.get(node.status) ?? 0) + 1) + const statuses = [...counts].map(([status, count]) => `${status}: ${count}`).join(", ") + const identity = `Definition: ${run.definitionId ?? "unknown"}${run.definitionRevision ? ` revision ${run.definitionRevision}` : ""}` + const execution = `Execution nodes: ${nodes.length} total${statuses ? ` (${statuses})` : ""}` + const usage = run.usage + ? `Usage: ${run.usage.tokens} tokens (input ${run.usage.inputTokens}, output ${run.usage.outputTokens}, reasoning ${run.usage.reasoningTokens}, cache read ${run.usage.cacheReadTokens}, cache write ${run.usage.cacheWriteTokens}); cost ${run.usage.cost}` + : "Usage: unavailable" + return `${identity}\n${execution}\n${usage}` +} + +function statusGuidance(status: WorkflowStatus) { + if (status === "waiting_for_review") return "Human approval is required in the CodeNomad UI before the workflow can continue or complete; plugin credentials cannot approve it." + if (status === "waiting_for_input") return "Human input is required in the CodeNomad UI; plugin credentials cannot answer it." + if (status === "pausing") return "The workflow is pausing. Monitor it in the CodeNomad UI." + if (status === "paused") return "The workflow is paused. Only a human can resume it in the CodeNomad UI." + if (status === "recovery_required") return "Recovery confirmation is required from a human in the CodeNomad UI; plugin credentials cannot confirm recovery." + return "" +} + +function describePendingGate(run: WorkflowRun) { + const gate = run.pendingGate + if (!gate) return "" + const action = gate.gate === "approval" ? "Approve or reject" : "Provide the requested input" + const restriction = gate.gate === "approval" ? "cannot approve this gate" : "cannot answer this gate" + const schema = gate.inputSchema ? `\nExpected input schema:\n${JSON.stringify(gate.inputSchema, null, 2)}` : "" + return `Pending ${gate.gate} gate (${gate.executionNodeId}): ${gate.prompt}${schema}\n${action} as a human in the CodeNomad UI; plugin credentials ${restriction}.` +} + +export function describeWorkflowDefinitions(definitions: WorkflowDefinitionRecord[]) { + if (definitions.length === 0) return "No saved CodeNomad workflow definitions found." + return definitions.map((record) => [ + `${record.id} | revision ${record.revision} | ${record.definition.name}`, + record.definition.description, + ].filter(Boolean).join(" | ")).join("\n") +} + +export function describeWorkflowDefinition(record: WorkflowDefinitionRecord) { + return [ + `Workflow definition ${record.id}`, + `Name: ${record.definition.name}`, + `Revision: ${record.revision}`, + record.definition.description ? `Description: ${record.definition.description}` : "", + `Canonical definition:\n${record.canonical}`, + ].filter(Boolean).join("\n") +} + +type WorkflowRequester = Pick, "requestJson"> + +export function createWorkflowTools(config: CodeNomadConfig, requester: WorkflowRequester = createCodeNomadRequester(config)) { + const request = (path: string, init?: RequestInit) => requester.requestJson(`/workflow-runs${path}`, init) + const requestDefinition = (path: string, init?: RequestInit) => requester.requestJson(`/workflow-definitions${path}`, init) + + return { + list_codenomad_workflow_definitions: tool({ + description: "List saved workflow definitions available to start at their current revision.", + args: {}, + async execute() { + const response = await requestDefinition<{ definitions: WorkflowDefinitionRecord[] }>("") + return describeWorkflowDefinitions(response.definitions) + }, + }), + get_codenomad_workflow_definition: tool({ + description: "Inspect the current revision of a saved workflow definition. This tool cannot inspect historical revisions.", + args: { definition_id: tool.schema.string().describe("Saved workflow definition ID") }, + async execute(args) { + const record = await requestDefinition(`/${encodeURIComponent(args.definition_id)}`) + return describeWorkflowDefinition(record) + }, + }), + start_codenomad_workflow_definition: tool({ + description: "Start the current revision of a saved workflow definition. Approval and input gates require a human in the CodeNomad UI.", + args: { + definition_id: tool.schema.string().describe("Saved workflow definition ID"), + objective: tool.schema.string().optional().describe("Optional objective; defaults to the definition name"), + inputs_json: tool.schema.string().optional().describe("Optional workflow inputs as a JSON object"), + }, + async execute(args, context) { + const inputs = parseWorkflowInputs(args.inputs_json) + context.abort.throwIfAborted() + const runId = randomUUID() + let run: WorkflowRun | undefined + let failure: unknown + let failed = false + try { + const response = await requestDefinition(`/${encodeURIComponent(args.definition_id)}/start`, { + method: "POST", + signal: AbortSignal.any([context.abort, AbortSignal.timeout(START_REQUEST_TIMEOUT_MS)]), + body: JSON.stringify({ + runId, + ...(args.objective ? { objective: args.objective } : {}), + ...(inputs ? { inputs } : {}), + }), + }) + run = validateStartedWorkflowRun(response, runId) + } catch (error) { + failed = true + failure = error + } + if (!run && !failed) { + failed = true + failure = new Error(`Workflow ${runId} start returned no run`) + } + if (failed || context.abort.aborted) { + try { + await request(`/${encodeURIComponent(runId)}/cancel`, { + method: "POST", + signal: AbortSignal.timeout(START_CANCEL_TIMEOUT_MS), + }) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Workflow ${runId} may have started but cancellation failed: ${message}`) + } + if (context.abort.aborted) { + const error = new Error("Workflow start aborted") + error.name = "AbortError" + throw error + } + throw failure + } + if (!run) throw failure + return `${summarize(run)}\nStarted the current saved definition revision. Only a human can manage approval, input, pause/resume, and recovery actions in the CodeNomad UI.` + }, + }), + list_codenomad_workflows: tool({ + description: "List workflow runs managed by CodeNomad for this workspace.", + args: {}, + async execute() { + const response = await request<{ runs: WorkflowRun[] }>("") + if (response.runs.length === 0) return "No CodeNomad workflow runs found." + return response.runs.map((run) => run.definitionId || run.executionNodes + ? `Objective: ${run.objective}\n${summarize(run)}` + : `${run.id} | ${run.status} | ${run.objective}`).join("\n\n") + }, + }), + get_codenomad_workflow: tool({ + description: "Inspect one CodeNomad workflow run and its role sessions.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args) { + return details(await request(`/${encodeURIComponent(args.run_id)}`)) + }, + }), + approve_codenomad_workflow: tool({ + description: + "Show the pending approval details. Only a user in the CodeNomad Workflows panel can approve and continue the workflow.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args) { + const run = await request(`/${encodeURIComponent(args.run_id)}`) + return `${details(run)}\nApproval was not applied. Ask the user to approve this run in the CodeNomad Workflows panel.` + }, + }), + cancel_codenomad_workflow: tool({ + description: "Cancel a running or review-pending CodeNomad workflow.", + args: { run_id: tool.schema.string().describe("Workflow run ID") }, + async execute(args, context) { + const run = await request(`/${encodeURIComponent(args.run_id)}/cancel`, { method: "POST", signal: context.abort }) + return summarize(run) + }, + }), + } +} diff --git a/packages/opencode-plugin/tsconfig.json b/packages/opencode-plugin/tsconfig.json index 09a866276..a7da85a5f 100644 --- a/packages/opencode-plugin/tsconfig.json +++ b/packages/opencode-plugin/tsconfig.json @@ -13,5 +13,5 @@ "types": ["node"] }, "include": ["plugin/**/*.ts"], - "exclude": ["dist", "node_modules"] + "exclude": ["dist", "node_modules", "plugin/**/*.test.ts"] } diff --git a/packages/server/src/api-types.ts b/packages/server/src/api-types.ts index d61b12f0e..72b5f4b3c 100644 --- a/packages/server/src/api-types.ts +++ b/packages/server/src/api-types.ts @@ -16,6 +16,8 @@ export type WorkspaceStatus = "starting" | "ready" | "stopped" | "error" export interface WorkspaceDescriptor { id: string + /** Stable across desktop restore; distinct for force-created instances of the same path. */ + lineageId?: string /** Correlates creation events with the client request that initiated them. */ requestId?: string /** Absolute path on the server host. */ @@ -39,6 +41,7 @@ export interface WorkspaceDescriptor { export interface WorkspaceCreateRequest { path: string + lineageId?: string name?: string binaryPath?: string requestId?: string @@ -293,6 +296,332 @@ export interface InstanceStreamEvent { [key: string]: unknown } +export type WorkflowRunStatus = + | "running" + | "pausing" + | "paused" + | "waiting_for_review" + | "waiting_for_input" + | "completed" + | "failed" + | "cancelled" + | "interrupted" + | "recovery_required" +export type WorkflowStepStatus = "pending" | "running" | "completed" | "failed" | "cancelled" + +export interface WorkflowModelSelection { + providerID: string + modelID: string +} + +export interface WorkflowStageConfig { + id: string + title: string + instructions: string + agent?: string + model?: WorkflowModelSelection + requiresApproval?: boolean +} + +export interface WorkflowValueRef { + /** Dot path rooted at inputs, nodes, or vars, for example `nodes.plan.output.steps`. */ + $ref: string +} + +export type WorkflowValue = null | boolean | number | string | WorkflowValueRef | WorkflowValue[] | { + [key: string]: WorkflowValue +} + +export type WorkflowCondition = boolean | { + value: WorkflowValue + equals?: WorkflowValue + notEquals?: WorkflowValue + exists?: boolean + truthy?: boolean +} + +export interface WorkflowRetryPolicy { + maxAttempts: number + delayMs?: number + /** Required when maxAttempts > 1 because an admitted operation may have completed before an error was observed. */ + idempotent?: boolean +} + +export interface WorkflowNodeBase { + id: string + title?: string + if?: WorkflowCondition +} + +export interface WorkflowSequenceNode extends WorkflowNodeBase { + type: "sequence" + steps: WorkflowNode[] +} + +export interface WorkflowParallelNode extends WorkflowNodeBase { + type: "parallel" + branches: WorkflowNode[] + maxConcurrency?: number +} + +export interface WorkflowForeachNode extends WorkflowNodeBase { + type: "foreach" + items: WorkflowValue + item: string + body: WorkflowNode + maxItems: number + maxConcurrency?: number +} + +export interface WorkflowRepeatNode extends WorkflowNodeBase { + type: "repeat" + body: WorkflowNode + maxIterations: number + while?: WorkflowCondition + onExhausted?: "complete" | "fail" +} + +export interface WorkflowAgentNode extends WorkflowNodeBase { + type: "agent" + instructions: string + context?: WorkflowValue + /** Nodes sharing a key reuse one durable OpenCode session and serialize their prompts. */ + sessionKey?: string + agent?: string + model?: WorkflowModelSelection + /** Omitted tools inherit the agent's normal OpenCode tool access. */ + tools?: string[] + outputSchema?: Record + retry?: WorkflowRetryPolicy + timeoutMs?: number +} + +export interface WorkflowShellNode extends WorkflowNodeBase { + type: "shell" + command: string + agent: string + model?: WorkflowModelSelection + retry?: WorkflowRetryPolicy + timeoutMs?: number +} + +export interface WorkflowGateNode extends WorkflowNodeBase { + type: "gate" + gate: "approval" | "input" + prompt: string + inputSchema?: Record +} + +export interface WorkflowSavedCallNode extends WorkflowNodeBase { + type: "workflow" + definitionId: string + /** Omitted definitions are resolved and pinned to the current revision when the run is admitted. */ + definitionRevision?: number + inputs?: Record +} + +export interface WorkflowBranchNode extends WorkflowNodeBase { + type: "condition" + condition: WorkflowCondition + then: WorkflowNode + else?: WorkflowNode +} + +export type WorkflowNode = + | WorkflowSequenceNode + | WorkflowParallelNode + | WorkflowForeachNode + | WorkflowRepeatNode + | WorkflowAgentNode + | WorkflowShellNode + | WorkflowGateNode + | WorkflowBranchNode + | WorkflowSavedCallNode + +export interface WorkflowBudget { + /** Stops admitting actions once observed cost reaches this value; provider reporting may let the final admitted action overshoot. */ + maxCost?: number + /** Stops admitting actions once observed tokens reach this value; provider reporting may let the final admitted action overshoot. */ + maxTokens?: number +} + +export interface WorkflowDefinitionV1 { + version: 1 + id: string + name: string + description?: string + root: WorkflowNode + budget?: WorkflowBudget + maxConcurrency?: number + maxExpandedNodes?: number +} + +export interface WorkflowDefinitionRecord { + id: string + revision: number + definition: WorkflowDefinitionV1 + canonical: string + createdAt: string + updatedAt: string +} + +export interface WorkflowSavedDefinitionSnapshot { + id: string + revision: number + definition: WorkflowDefinitionV1 +} + +export type WorkflowRunWorktreePolicy = + | { mode: "current" } + | { mode: "existing"; slug: string } + | { mode: "new"; slug: string } + +export interface WorkflowRunWorktreeSelection { + policy: WorkflowRunWorktreePolicy + sourceWorkspaceId: string + sourceWorkspaceLineageId: string + sourceWorkspacePath: string + workspaceId: string + directory: string + slug?: string + branch?: string + created: boolean +} + +export interface WorkflowUsage { + cost: number + tokens: number + inputTokens: number + outputTokens: number + reasoningTokens: number + cacheReadTokens: number + cacheWriteTokens: number +} + +export type WorkflowExecutionNodeStatus = + | "pending" + | "running" + | "waiting" + | "completed" + | "skipped" + | "failed" + | "cancelled" + | "interrupted" + +export interface WorkflowExecutionNode { + id: string + instanceKey: string + /** Identifies one root or saved-definition invocation for nodes. reference isolation. */ + definitionInvocationKey?: string + definitionNodeId: string + type: WorkflowNode["type"] + status: WorkflowExecutionNodeStatus + attempt: number + parentInstanceKey?: string + sessionIds?: string[] + output?: unknown + outputTruncated?: boolean + error?: string + usage?: WorkflowUsage + startedAt?: string + completedAt?: string +} + +export interface WorkflowPendingGate { + executionNodeId: string + definitionNodeId: string + gate: "approval" | "input" + prompt: string + inputSchema?: Record +} + +export interface WorkflowRunStep extends WorkflowStageConfig { + status: WorkflowStepStatus + sessionId?: string + output?: unknown + outputTruncated?: boolean + error?: string + startedAt?: string + completedAt?: string +} + +export interface WorkflowExecutorLease { + ownerToken: string + fence: number + heartbeatAt: string + expiresAt: string + hostname?: string + pid?: number + processStart?: string + bootId?: string +} + +export interface WorkflowRun { + id: string + workspaceId: string + workspaceLineageId: string + workspacePath: string + initiatorSessionId?: string + objective: string + status: WorkflowRunStatus + rootSessionId?: string + activeStepId?: string + pendingReviewStepId?: string + steps: WorkflowRunStep[] + revision?: number + executorFence?: number + executorLease?: WorkflowExecutorLease + definitionId?: string + definitionRevision?: number + definitionSnapshot?: WorkflowDefinitionV1 + savedDefinitionSnapshots?: WorkflowSavedDefinitionSnapshot[] + sessionBindings?: Record + worktreeSelection?: WorkflowRunWorktreeSelection + inputs?: Record + executionNodes?: WorkflowExecutionNode[] + pendingGate?: WorkflowPendingGate + usage?: WorkflowUsage + pauseRequested?: boolean + error?: string + createdAt: string + updatedAt: string +} + +export interface WorkflowRunCreateRequest { + workspaceId: string + initiatorSessionId?: string + objective: string + stages: WorkflowStageConfig[] +} + +export interface WorkflowDefinitionRunCreateRequest { + runId?: string + workspaceId: string + initiatorSessionId?: string + objective?: string + definitionId: string + definitionRevision?: number + inputs?: Record + worktree?: WorkflowRunWorktreePolicy +} + +export type WorkflowRunStartRequest = WorkflowRunCreateRequest | WorkflowDefinitionRunCreateRequest + +export type WorkflowDefinitionPayload = + | { source: string; definition?: never } + | { definition: WorkflowDefinitionV1; source?: never } + +export type WorkflowDefinitionUpdateRequest = WorkflowDefinitionPayload & { expectedRevision: number } + +export interface WorkflowGateAnswerRequest { + executionNodeId: string + answer: unknown +} + +export type WorkflowResumeRequest = + | { confirmRecovery: true; expectedRevision: number } + | { confirmRecovery?: false; expectedRevision?: never } + export type SideCarKind = "port" export type SideCarPrefixMode = "strip" | "preserve" diff --git a/packages/server/src/clients/connection-manager.ts b/packages/server/src/clients/connection-manager.ts index 7eaa426ff..45c28f8ef 100644 --- a/packages/server/src/clients/connection-manager.ts +++ b/packages/server/src/clients/connection-manager.ts @@ -67,7 +67,9 @@ export class ClientConnectionManager { this.connections.set(key, connection) this.logger.debug({ clientId: input.clientId, connectionId: input.connectionId }, "Client connected") this.notify({ type: "connected", connection }) - return () => this.disconnect(key, "closed") + return () => { + if (this.connections.get(key) === connection) this.disconnect(key, "closed") + } } pong(input: ClientConnectionRef): boolean { diff --git a/packages/server/src/events/bus.test.ts b/packages/server/src/events/bus.test.ts index 71757ceb2..26a9ce245 100644 --- a/packages/server/src/events/bus.test.ts +++ b/packages/server/src/events/bus.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" -import { EventBus } from "./bus" +import { EventBus, type EventReplayGap } from "./bus" import type { WorkspaceEventPayload } from "../api-types" describe("event bus instance status replay", () => { @@ -43,3 +43,115 @@ describe("event bus instance status replay", () => { assert.deepEqual(replayed, []) }) }) + +describe("event bus sequence replay", () => { + it("bounds replay and preserves replay-to-live publish order", () => { + const bus = new EventBus(undefined, 2, Infinity, "test") + for (const sequence of [1, 2, 3]) { + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence } } as never) + } + + const received: Array<{ cursor?: string; sequence: number }> = [] + bus.onEvent((event, cursor) => { + const sequence = (event as never as { entry: { sequence: number } }).entry.sequence + received.push({ cursor, sequence }) + if (sequence === 2) { + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 4 } } as never) + } + }, "test:1") + + assert.deepEqual(received, [ + { cursor: "test:2", sequence: 2 }, + { cursor: "test:3", sequence: 3 }, + { cursor: "test:4", sequence: 4 }, + ]) + }) + + it("signals overflow before live delivery instead of replaying a partial window", () => { + const bus = new EventBus(undefined, 2, Infinity, "test") + for (const sequence of [1, 2, 3]) { + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence } } as never) + } + + const received: string[] = [] + bus.onEvent( + (event, cursor) => received.push(`${cursor}:${(event as never as { entry: { sequence: number } }).entry.sequence}`), + "test:0", + (gap) => { + received.push(`reset:${gap.requestedCursor}:${gap.earliestAvailableCursor}:${gap.latestCursor}`) + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 4 } } as never) + }, + ) + + assert.deepEqual(received, ["reset:test:0:test:2:test:3", "test:4:4"]) + }) + + it("rejects a cursor from a previous server epoch", () => { + const previous = new EventBus(undefined, 1, Infinity, "old") + previous.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: {} } as never) + const bus = new EventBus(undefined, 1, Infinity, "new") + let gap: unknown + + bus.onEvent(() => undefined, previous.latestCursor, (value) => { + gap = value + }) + + assert.deepEqual(gap, { + requestedCursor: "old:1", + earliestAvailableCursor: "new:1", + latestCursor: "new:0", + }) + }) + + it("delivers re-entrant publications to every listener in id order", () => { + const bus = new EventBus(undefined, 10, Infinity, "test") + const received: string[] = [] + bus.onEvent((event) => { + if ((event as any).entry.sequence === 1) { + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 2 } } as never) + } + }, bus.latestCursor) + bus.onEvent((event, cursor) => { + received.push(`${cursor}:${(event as any).entry.sequence}`) + }, bus.latestCursor) + + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 1 } } as never) + + assert.deepEqual(received, ["test:1:1", "test:2:2"]) + }) + + it("does not replay a queued re-entrant event twice to a new subscriber", () => { + const bus = new EventBus(undefined, 10, Infinity, "test") + const received: string[] = [] + bus.onEvent((event) => { + if ((event as any).entry.sequence !== 1) return + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 2 } } as never) + bus.onEvent((replayed, cursor) => { + received.push(`${cursor}:${(replayed as any).entry.sequence}`) + }, "test:0") + }, bus.latestCursor) + + bus.publish({ type: "workspace.log", workspaceId: "workspace-1", entry: { sequence: 1 } } as never) + + assert.deepEqual(received, ["test:1:1", "test:2:2"]) + }) + + it("bounds replay by serialized bytes", () => { + const bus = new EventBus(undefined, 100, 250, "test") + for (const sequence of [1, 2]) { + bus.publish({ + type: "workspace.log", + workspaceId: "workspace-1", + entry: { sequence, message: "x".repeat(120) }, + } as never) + } + let gap: EventReplayGap | undefined + + bus.onEvent(() => undefined, "test:0", (value) => { + gap = value + }) + + assert.equal(gap?.latestCursor, "test:2") + assert.notEqual(gap?.earliestAvailableCursor, "test:1") + }) +}) diff --git a/packages/server/src/events/bus.ts b/packages/server/src/events/bus.ts index 637aad1d2..d71ebb0cb 100644 --- a/packages/server/src/events/bus.ts +++ b/packages/server/src/events/bus.ts @@ -1,15 +1,50 @@ import { EventEmitter } from "events" +import { randomUUID } from "node:crypto" import { WorkspaceEventPayload } from "../api-types" import { Logger } from "../logger" +export interface EventReplayGap { + requestedCursor: string + earliestAvailableCursor: string + latestCursor: string +} + +const DEFAULT_REPLAY_BYTE_LIMIT = 8 * 1024 * 1024 + export class EventBus extends EventEmitter { private readonly instanceStatuses = new Map>() + private readonly replay: Array<{ id: number; event: WorkspaceEventPayload; bytes: number }> = [] + private readonly deliveryQueue: Array<{ id: number; event: WorkspaceEventPayload }> = [] + private replayBytes = 0 + private nextEventId = 0 + private latestDeliveredEventId = 0 + private currentDeliveryId: number | undefined + private dispatching = false - constructor(private readonly logger?: Logger) { + constructor( + private readonly logger?: Logger, + private readonly replayLimit = 1_000, + private readonly replayByteLimit = DEFAULT_REPLAY_BYTE_LIMIT, + private readonly epoch: string = randomUUID(), + ) { super() } + get latestCursor(): string { + return this.cursor(this.currentDeliveryId ?? this.latestDeliveredEventId) + } + publish(event: WorkspaceEventPayload): boolean { + const sequenced = { + id: ++this.nextEventId, + event, + bytes: Buffer.byteLength(JSON.stringify(event)) + this.epoch.length + 16, + } + this.replay.push(sequenced) + this.replayBytes += sequenced.bytes + while (this.replay.length > this.replayLimit || this.replayBytes > this.replayByteLimit) { + this.replayBytes -= this.replay.shift()!.bytes + } if (event.type === "instance.eventStatus") { const terminal = event.status === "disconnected" && (event.reason === "workspace stopped" || event.reason === "workspace error") @@ -25,11 +60,46 @@ export class EventBus extends EventEmitter { this.logger.trace({ event }, "Workspace event payload") } } - return super.emit(event.type, event) + const hadListeners = this.listenerCount(event.type) > 0 + this.deliveryQueue.push(sequenced) + if (this.dispatching) return hadListeners + + let delivered = false + this.dispatching = true + try { + while (this.deliveryQueue.length > 0) { + const next = this.deliveryQueue.shift()! + this.currentDeliveryId = next.id + delivered = super.emit(next.event.type, next.event, this.cursor(next.id)) || delivered + this.latestDeliveredEventId = next.id + this.currentDeliveryId = undefined + } + } finally { + this.currentDeliveryId = undefined + this.dispatching = false + } + return delivered } - onEvent(listener: (event: WorkspaceEventPayload) => void) { - const handler = (event: WorkspaceEventPayload) => listener(event) + onEvent( + listener: (event: WorkspaceEventPayload, cursor?: string) => void, + afterCursor?: string, + onReplayGap?: (gap: EventReplayGap) => void, + ) { + const replayBoundary = this.currentDeliveryId ?? this.latestDeliveredEventId + const replaySnapshot = this.replay.filter((entry) => entry.id <= replayBoundary) + const earliestAvailableId = replaySnapshot[0]?.id ?? replayBoundary + 1 + const afterId = afterCursor === undefined ? undefined : this.parseCursor(afterCursor) + const replayGap = afterCursor !== undefined + && (afterId === undefined || afterId < earliestAvailableId - 1 || afterId > replayBoundary) + const pendingLive: Array<{ id: number; event: WorkspaceEventPayload }> = [] + let replaying = true + const handler = (event: WorkspaceEventPayload, cursor: string) => { + const id = this.parseCursor(cursor) + if (id === undefined) return + if (replaying) pendingLive.push({ event, id }) + else listener(event, cursor) + } this.on("workspace.created", handler) this.on("workspace.started", handler) this.on("workspace.error", handler) @@ -44,7 +114,24 @@ export class EventBus extends EventEmitter { this.on("instance.eventStatus", handler) this.on("yolo.stateChanged", handler) this.on("yolo.autoAccepted", handler) - for (const status of this.instanceStatuses.values()) listener(status) + if (afterCursor === undefined) { + for (const status of this.instanceStatuses.values()) listener(status) + } else if (replayGap) { + onReplayGap?.({ + requestedCursor: afterCursor, + earliestAvailableCursor: this.cursor(earliestAvailableId), + latestCursor: this.cursor(replayBoundary), + }) + } else { + for (const entry of replaySnapshot) { + if (entry.id > afterId!) listener(entry.event, this.cursor(entry.id)) + } + } + for (let index = 0; index < pendingLive.length; index += 1) { + const entry = pendingLive[index]! + listener(entry.event, this.cursor(entry.id)) + } + replaying = false return () => { this.off("workspace.created", handler) this.off("workspace.started", handler) @@ -62,4 +149,17 @@ export class EventBus extends EventEmitter { this.off("yolo.autoAccepted", handler) } } + + private cursor(id: number): string { + return `${this.epoch}:${id}` + } + + private parseCursor(cursor: string): number | undefined { + const prefix = `${this.epoch}:` + if (!cursor.startsWith(prefix)) return undefined + const sequence = cursor.slice(prefix.length) + if (!/^\d+$/.test(sequence)) return undefined + const id = Number(sequence) + return Number.isSafeInteger(id) ? id : undefined + } } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 3993ed7a6..76ecb78a8 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -36,6 +36,7 @@ import { createServerShutdownHandler, orchestrateServerShutdown, type ServerShut import { AutoAcceptManager } from "./permissions/auto-accept-manager" import { createOpencodePermissionReplier } from "./permissions/opencode-replier" import { createOpencodeYoloPersistence } from "./permissions/opencode-yolo-metadata" +import { WorkflowManager } from "./workflows/manager" const require = createRequire(import.meta.url) @@ -374,6 +375,14 @@ async function main() { logger: workspaceLogger, getServerBaseUrl: () => serverMeta.localUrl, nodeExtraCaCertsPath, + workspaceLeaseDir: path.join(configDir, "workspace-leases"), + }) + const workflowManager = new WorkflowManager({ + workspaceManager, + eventBus, + storageDir: path.join(configDir, "workflow-runs"), + definitionsDir: path.join(configDir, "workflow-definitions"), + logger: logger.child({ component: "workflows" }), }) const fileSystemBrowser = new FileSystemBrowser({ rootDir: options.rootDir, @@ -499,6 +508,7 @@ async function main() { remoteProxySessionManager, yoloManager, sessionMetadataPersistence, + workflowManager, uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR, uiDevServerUrl: uiResolution.uiDevServerUrl, logger, @@ -528,6 +538,7 @@ async function main() { remoteProxySessionManager, yoloManager, sessionMetadataPersistence, + workflowManager, uiStaticDir: uiResolution.uiStaticDir ?? DEFAULT_UI_STATIC_DIR, uiDevServerUrl: undefined, logger, @@ -623,6 +634,7 @@ async function main() { orchestrateServerShutdown( { stopInstanceEventBridge: () => instanceEventBridge.shutdown(), + stopWorkflowRuns: () => workflowManager.shutdown(), stopSidecars: () => sidecarManager.shutdown(), stopClientConnections: () => clientConnectionManager.shutdown(), stopRemoteProxySessions: () => remoteProxySessionManager.shutdown(), diff --git a/packages/server/src/server/http-server.test.ts b/packages/server/src/server/http-server.test.ts new file mode 100644 index 000000000..da221c518 --- /dev/null +++ b/packages/server/src/server/http-server.test.ts @@ -0,0 +1,141 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import pino from "pino" + +import { EventBus } from "../events/bus" +import { createHttpServer } from "./http-server" + +test("browser origins and plugin callbacks use separate security gates", async () => { + const logger = pino({ level: "silent" }) + const workspace = { + id: "workspace", + path: process.cwd(), + status: "ready", + proxyPath: "/workspaces/workspace/instance", + binaryId: "opencode", + binaryLabel: "opencode", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } + let deletionGuardWired = false + const workspaceManager = { + get: (id: string) => id === workspace.id ? workspace : undefined, + list: () => [workspace], + getPluginCallbackAuthorizationHeader: (id: string) => id === workspace.id ? "Bearer callback-secret" : undefined, + getInstanceAuthorizationHeader: () => "Basic shared-opencode-secret", + setDeletionGuard: () => { deletionGuardWired = true }, + } + const authManager = { + isTokenBootstrapEnabled: () => false, + isLoopbackRequest: () => true, + getSessionFromRequest: (request: { headers: { cookie?: string } }) => request.headers.cookie === "session=valid" ? { id: "session" } : null, + } + const server = createHttpServer({ + bindHost: "0.0.0.0", + bindPort: 0, + defaultPort: 0, + protocol: "http", + workspaceManager, + settings: {}, + fileSystemBrowser: {}, + eventBus: new EventBus(), + serverMeta: { + localUrl: "http://localhost:4000", + remoteUrl: "http://192.168.1.2:4000", + addresses: [], + }, + instanceStore: {}, + speechService: {}, + sidecarManager: {}, + previewManager: {}, + authManager, + clientConnectionManager: { pong: () => true, register: () => () => undefined }, + pluginChannel: {}, + voiceModeManager: {}, + remoteProxySessionManager: {}, + yoloManager: {}, + sessionMetadataPersistence: {}, + workflowManager: { list: async () => [] }, + uiStaticDir: "", + uiDevServerUrl: "http://localhost:3000", + logger, + } as never) + assert.equal(deletionGuardWired, true) + await server.instance.ready() + try { + const hostile = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", origin: "https://attacker.example" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(hostile.statusCode, 403) + assert.equal(hostile.headers["access-control-allow-origin"], undefined) + + const trusted = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", origin: "http://localhost:3000" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(trusted.statusCode, 204) + assert.equal(trusted.headers["access-control-allow-origin"], "http://localhost:3000") + + const alias = await server.instance.inject({ + method: "POST", + url: "/api/client-connections/pong", + headers: { cookie: "session=valid", host: "codenomad.local", origin: "http://codenomad.local" }, + payload: { clientId: "client", connectionId: "connection" }, + }) + assert.equal(alias.statusCode, 204) + + for (const headers of [ + {}, + { cookie: "session=valid" }, + { authorization: "Basic shared-opencode-secret" }, + ]) { + const rejected = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers, + payload: { type: "test.event" }, + }) + assert.equal(rejected.statusCode, 401) + } + + const callback = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + assert.equal(callback.statusCode, 204) + + const noncanonicalCallback = await server.instance.inject({ + method: "POST", + url: "/workspaces/workspace/plugin//event", + headers: { cookie: "session=valid" }, + payload: { type: "test.event" }, + }) + assert.equal(noncanonicalCallback.statusCode, 404) + + const trustedSsePending = server.instance.inject({ + method: "GET", + url: "/api/events?clientId=trusted&connectionId=trusted", + headers: { cookie: "session=valid", origin: "http://localhost:3000" }, + }) + const hostileSsePending = server.instance.inject({ + method: "GET", + url: "/api/events?clientId=hostile&connectionId=hostile", + headers: { cookie: "session=valid", origin: "https://attacker.example" }, + }) + await new Promise((resolve) => setImmediate(resolve)) + await server.stop() + const [trustedSse, hostileSse] = await Promise.all([trustedSsePending, hostileSsePending]) + assert.equal(trustedSse.headers["access-control-allow-origin"], "http://localhost:3000") + assert.equal(trustedSse.headers["access-control-allow-credentials"], "true") + assert.equal(hostileSse.headers["access-control-allow-origin"], undefined) + } finally { + if (server.instance.server.listening) await server.stop() + } +}) diff --git a/packages/server/src/server/http-server.ts b/packages/server/src/server/http-server.ts index 44e855b27..71d17ae31 100644 --- a/packages/server/src/server/http-server.ts +++ b/packages/server/src/server/http-server.ts @@ -8,7 +8,7 @@ import path from "path" import { connect as connectTls, type TLSSocket } from "tls" import { fetch, type Headers } from "undici" import type { Logger } from "../logger" -import { WorkspaceManager } from "../workspaces/manager" +import { WorkspaceDeletionBlockedError, WorkspaceManager } from "../workspaces/manager" import type { SettingsService } from "../settings/service" import { FileSystemBrowser } from "../filesystem/browser" @@ -47,6 +47,8 @@ import type { SideCarManager } from "../sidecars/manager" import type { PreviewManager } from "../previews/manager" import type { RemoteProxySessionManager } from "./remote-proxy" import { createOpenCodeUpdateService } from "../opencode-update/service" +import type { WorkflowManager } from "../workflows/manager" +import { registerWorkflowRoutes } from "./routes/workflows" interface HttpServerDeps { bindHost: string @@ -71,6 +73,7 @@ interface HttpServerDeps { remoteProxySessionManager: RemoteProxySessionManager yoloManager: AutoAcceptManager sessionMetadataPersistence: OpencodeYoloPersistence + workflowManager: WorkflowManager uiStaticDir: string uiDevServerUrl?: string logger: Logger @@ -89,6 +92,16 @@ export function shouldRetryPreferredPort(error: unknown, autoPortRequested: bool } export function createHttpServer(deps: HttpServerDeps) { + deps.workspaceManager.setDeletionGuard?.((workspace, operation) => + deps.workflowManager.withWorkspaceOwnershipLease({ + id: workspace.id, + lineageId: workspace.lineageId, + path: workspace.path, + }, async (owned) => { + if (owned) throw new WorkspaceDeletionBlockedError(workspace.id) + return operation() + })) + // Fastify's type-level RawServer inference gets noisy when toggling HTTP vs HTTPS. // We keep the runtime behavior correct and cast the instance to a generic FastifyInstance. const app = Fastify( @@ -136,12 +149,9 @@ export function createHttpServer(deps: HttpServerDeps) { done() }) - const allowedDevOrigins = new Set(["http://localhost:3000", "http://127.0.0.1:3000"]) - const isLoopbackHost = (host: string) => host === "127.0.0.1" || host === "::1" || host.startsWith("127.") - - const getSelfOrigins = (): Set => { + const getAllowedOrigins = (): Set => { const origins = new Set() - const candidates: Array = [deps.serverMeta.localUrl, deps.serverMeta.remoteUrl] + const candidates: Array = [deps.serverMeta.localUrl, deps.serverMeta.remoteUrl, deps.uiDevServerUrl] for (const candidate of candidates) { if (!candidate) continue try { @@ -167,24 +177,11 @@ export function createHttpServer(deps: HttpServerDeps) { return } - const selfOrigins = getSelfOrigins() - if (selfOrigins.has(origin)) { + if (getAllowedOrigins().has(origin)) { cb(null, true) return } - if (allowedDevOrigins.has(origin)) { - cb(null, true) - return - } - - // When we bind to a non-loopback host (e.g., 0.0.0.0 or LAN IP), allow cross-origin UI access. - if (deps.bindHost === "0.0.0.0" || !isLoopbackHost(deps.bindHost)) { - cb(null, true) - return - } - - cb(null, false) }, credentials: true, @@ -211,6 +208,25 @@ export function createHttpServer(deps: HttpServerDeps) { app.addHook("preHandler", (request, reply, done) => { const rawUrl = request.raw.url ?? request.url const pathname = (rawUrl.split("?")[0] ?? "").trim() + const session = deps.authManager.getSessionFromRequest(request) + const originHeader = request.headers.origin + const origin = Array.isArray(originHeader) ? originHeader[0] : originHeader + const fetchSiteHeader = request.headers["sec-fetch-site"] + const fetchSite = Array.isArray(fetchSiteHeader) ? fetchSiteHeader[0] : fetchSiteHeader + let requestOrigin: string | undefined + try { + if (request.headers.host) requestOrigin = new URL(`${request.protocol}://${request.headers.host}`).origin + } catch { + // Invalid Host values cannot establish a trusted browser origin. + } + if ( + session + && ["POST", "PUT", "PATCH", "DELETE"].includes(request.method) + && ((origin && origin !== requestOrigin && !getAllowedOrigins().has(origin)) || fetchSite === "cross-site") + ) { + reply.code(403).send({ error: "Cross-origin mutation forbidden" }) + return + } const publicApiPaths = new Set(["/api/auth/login", "/api/auth/token", "/api/auth/status", "/api/auth/logout"]) const publicPagePaths = new Set(["/login"]) @@ -228,23 +244,29 @@ export function createHttpServer(deps: HttpServerDeps) { return } - const session = deps.authManager.getSessionFromRequest(request) + const pluginMatch = pathname.match(/^\/workspaces\/([^/]+)\/plugin(?:\/|$)/) + const provided = Array.isArray(request.headers.authorization) + ? request.headers.authorization[0] + : request.headers.authorization + let pluginAuthorized = false + if (pluginMatch) { + const expected = deps.workspaceManager.getPluginCallbackAuthorizationHeader(pluginMatch[1] ?? "") + pluginAuthorized = Boolean(expected && provided && provided === expected) + } + const requiresPluginCapability = Boolean(pluginMatch) && ( + (request.method === "GET" && pathname.endsWith("/plugin/events")) + || (request.method === "POST" && pathname.endsWith("/plugin/event")) + ) + if (requiresPluginCapability && !pluginAuthorized) { + sendUnauthorized(request, reply) + return + } const requiresAuthForApi = pathname.startsWith("/api/") || pathname.startsWith("/workspaces/") || pathname.startsWith("/sidecars/") || pathname.startsWith("/previews/") if (requiresAuthForApi && !session) { - // Allow OpenCode plugin -> CodeNomad calls with per-instance basic auth. - const pluginMatch = pathname.match(/^\/workspaces\/([^/]+)\/plugin(?:\/|$)/) - if (pluginMatch) { - const workspaceId = pluginMatch[1] - const expected = deps.workspaceManager.getInstanceAuthorizationHeader(workspaceId) - const provided = Array.isArray(request.headers.authorization) - ? request.headers.authorization[0] - : request.headers.authorization - - if (expected && provided && provided === expected) { - done() - return - } + if (pluginAuthorized) { + done() + return } sendUnauthorized(request, reply) @@ -299,6 +321,7 @@ export function createHttpServer(deps: HttpServerDeps) { registerWorktreeRoutes(app, { workspaceManager: deps.workspaceManager, sessionMetadataPersistence: deps.sessionMetadataPersistence, + workflowManager: deps.workflowManager, }) registerStorageRoutes(app, { instanceStore: deps.instanceStore, @@ -332,6 +355,7 @@ export function createHttpServer(deps: HttpServerDeps) { }) registerBackgroundProcessRoutes(app, { backgroundProcessManager }) registerYoloRoutes(app, { yoloManager: deps.yoloManager }) + registerWorkflowRoutes(app, { workflowManager: deps.workflowManager }) registerInstanceProxyRoutes(app, { workspaceManager: deps.workspaceManager, logger: proxyLogger }) diff --git a/packages/server/src/server/routes/events.test.ts b/packages/server/src/server/routes/events.test.ts new file mode 100644 index 000000000..fc2fbcd86 --- /dev/null +++ b/packages/server/src/server/routes/events.test.ts @@ -0,0 +1,214 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import Fastify from "fastify" + +import { ClientConnectionManager } from "../../clients/connection-manager" +import { EventBus } from "../../events/bus" +import { registerEventRoutes } from "./events" + +test("SSE does not reflect request origins", async () => { + const app = Fastify({ logger: false }) + registerEventRoutes(app, { + eventBus: new EventBus(), + registerClient: (close) => { + setImmediate(close) + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + const response = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + headers: { origin: "https://attacker.example" }, + }) + + assert.equal(response.headers["access-control-allow-origin"], undefined) + assert.equal(response.headers["access-control-allow-credentials"], undefined) + await app.close() +}) + +test("SSE acknowledges its initial cursor after bootstrap statuses", async () => { + const app = Fastify({ logger: false }) + const eventBus = new EventBus(undefined, 1_000, Infinity, "test") + eventBus.publish({ type: "instance.eventStatus", instanceId: "workspace", status: "connected" }) + app.addHook("onRequest", (_request, reply, done) => { + const write = reply.raw.write.bind(reply.raw) + reply.raw.write = ((chunk: unknown, ...args: unknown[]) => { + const accepted = write(chunk, ...args as []) + if (!String(chunk).includes("instance.eventStatus")) return accepted + setImmediate(() => reply.raw.emit("drain")) + return false + }) as typeof reply.raw.write + done() + }) + registerEventRoutes(app, { + eventBus, + registerClient: (close) => { + setImmediate(close) + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + const response = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + }) + + assert.ok(response.payload.indexOf("instance.eventStatus") < response.payload.indexOf("codenomad.replay.cursor")) + assert.match(response.payload, /event: codenomad\.replay\.cursor\nid: test:1\ndata: \{\}\n\n/) + await app.close() +}) + +test("SSE drains an accepted backpressured frame before disconnecting", async () => { + const app = Fastify({ logger: false }) + const eventBus = new EventBus() + app.addHook("onRequest", (_request, reply, done) => { + const write = reply.raw.write.bind(reply.raw) + reply.raw.write = ((chunk: unknown, ...args: unknown[]) => { + const accepted = write(chunk, ...args as []) + if (String(chunk).includes("workflow.run.updated")) { + setImmediate(() => reply.raw.emit("drain")) + return false + } + return accepted + }) as typeof reply.raw.write + done() + }) + registerEventRoutes(app, { + eventBus, + registerClient: () => { + setImmediate(() => eventBus.publish({ + type: "instance.event", + instanceId: "workspace", + event: { type: "workflow.run.updated", properties: { run: { snapshot: "large" } } }, + } as never)) + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + const response = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + }).catch(() => undefined) + + assert.match(response?.payload ?? "", /workflow\.run\.updated/) + assert.equal(eventBus.listenerCount("instance.event"), 0) + await app.close() +}) + +test("SSE replays ordered events published behind a backpressured frame", async () => { + const app = Fastify({ logger: false }) + const eventBus = new EventBus(undefined, 1_000, Infinity, "test") + let requestCount = 0 + app.addHook("onRequest", (_request, reply, done) => { + requestCount += 1 + if (requestCount === 1) { + const write = reply.raw.write.bind(reply.raw) + reply.raw.write = ((chunk: unknown, ...args: unknown[]) => { + const accepted = write(chunk, ...args as []) + if (String(chunk).includes('"sequence":1')) { + setImmediate(() => reply.raw.emit("drain")) + return false + } + return accepted + }) as typeof reply.raw.write + } + done() + }) + registerEventRoutes(app, { + eventBus, + registerClient: (close) => { + if (requestCount === 1) { + setImmediate(() => { + for (const sequence of [1, 2, 3]) { + eventBus.publish({ + type: "instance.event", + instanceId: "workspace", + event: { type: "test.event", properties: { sequence } }, + } as never) + } + }) + } else { + setImmediate(close) + } + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + }).catch(() => undefined) + assert.equal(eventBus.listenerCount("instance.event"), 0) + + const replay = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + headers: { "last-event-id": "test:1" }, + }) + + assert.deepEqual( + [...replay.payload.matchAll(/id: (test:\d+)\ndata: .*?"sequence":(\d+)/g)].map((match) => [match[1], Number(match[2])]), + [["test:2", 2], ["test:3", 3]], + ) + assert.equal(eventBus.listenerCount("instance.event"), 0) + await app.close() +}) + +test("SSE signals an overflow gap before handing off to live events", async () => { + const app = Fastify({ logger: false }) + const eventBus = new EventBus(undefined, 2, Infinity, "test") + for (const sequence of [1, 2, 3]) { + eventBus.publish({ + type: "workspace.log", + workspaceId: "workspace", + entry: { sequence }, + } as never) + } + registerEventRoutes(app, { + eventBus, + registerClient: (close) => { + setImmediate(() => { + eventBus.publish({ + type: "workspace.log", + workspaceId: "workspace", + entry: { sequence: 4 }, + } as never) + close() + }) + return () => undefined + }, + logger: { debug: () => undefined, isLevelEnabled: () => false } as never, + connectionManager: { register: () => () => undefined } as never, + }) + + const response = await app.inject({ + method: "GET", + url: "/api/events?clientId=client&connectionId=connection", + headers: { "last-event-id": "test:0" }, + }) + + assert.match(response.payload, /event: codenomad\.replay\.reset\nid: test:3\ndata: \{"requestedCursor":"test:0","earliestAvailableCursor":"test:2","latestCursor":"test:3"\}/) + assert.doesNotMatch(response.payload, /"sequence":[23]/) + assert.ok(response.payload.indexOf("codenomad.replay.reset") < response.payload.indexOf('"sequence":4')) + await app.close() +}) + +test("an old SSE cleanup cannot unregister its replacement", () => { + const manager = new ClientConnectionManager({ debug: () => undefined, warn: () => undefined } as never) + const connection = { clientId: "client", connectionId: "connection" } + const unregisterOld = manager.register({ ...connection, close: () => undefined }) + manager.register({ ...connection, close: () => undefined }) + + unregisterOld() + assert.equal(manager.isConnected(connection), true) + manager.shutdown() +}) diff --git a/packages/server/src/server/routes/events.ts b/packages/server/src/server/routes/events.ts index 158266e12..cb9b85dee 100644 --- a/packages/server/src/server/routes/events.ts +++ b/packages/server/src/server/routes/events.ts @@ -1,6 +1,6 @@ -import { FastifyInstance } from "fastify" +import { FastifyInstance, type FastifyReply } from "fastify" import { z } from "zod" -import { EventBus } from "../../events/bus" +import { EventBus, type EventReplayGap } from "../../events/bus" import { WorkspaceEventPayload } from "../../api-types" import type { ClientConnectionManager } from "../../clients/connection-manager" import { Logger } from "../../logger" @@ -13,6 +13,7 @@ interface RouteDeps { } let nextClientId = 0 +const BACKPRESSURE_TIMEOUT_MS = 5_000 const ConnectionQuerySchema = z.object({ clientId: z.string().trim().min(1), @@ -27,40 +28,79 @@ export function registerEventRoutes(app: FastifyInstance, deps: RouteDeps) { app.get("/api/events", (request, reply) => { const clientId = ++nextClientId const connection = ConnectionQuerySchema.parse(request.query ?? {}) + const lastEventCursor = readLastEventCursor(request.headers["last-event-id"]) deps.logger.debug({ clientId }, "SSE client connected") - const origin = request.headers.origin ?? "*" - reply.raw.setHeader("Access-Control-Allow-Origin", origin) - reply.raw.setHeader("Access-Control-Allow-Credentials", "true") - reply.raw.setHeader("Content-Type", "text/event-stream") - reply.raw.setHeader("Cache-Control", "no-cache") - reply.raw.setHeader("Connection", "keep-alive") + reply.header("Content-Type", "text/event-stream") + reply.header("Cache-Control", "no-cache") + reply.header("Connection", "keep-alive") + copyReplyHeadersToRaw(reply) reply.raw.flushHeaders?.() reply.hijack() - const send = (event: WorkspaceEventPayload) => { + let closed = false + let blocked = false + let heartbeat: ReturnType | undefined + let backpressureTimeout: ReturnType | undefined + let drainListener: (() => void) | undefined + let unsubscribe: () => void = () => undefined + let bootstrapFrames: string[] | undefined = lastEventCursor === undefined ? [] : undefined + + const close = (discardBufferedData = false) => { + if (closed) return + closed = true + if (heartbeat) clearInterval(heartbeat) + if (backpressureTimeout) clearTimeout(backpressureTimeout) + if (drainListener) reply.raw.off("drain", drainListener) + unsubscribe() + if (discardBufferedData || blocked) reply.raw.destroy() + else reply.raw.end?.() + deps.logger.debug({ clientId }, "SSE client disconnected") + } + + const write = (payload: string) => { + if (closed || blocked) return + if (reply.raw.write(payload)) return + blocked = true + drainListener = () => { + drainListener = undefined + blocked = false + close() + } + reply.raw.once("drain", drainListener) + backpressureTimeout = setTimeout(() => close(true), BACKPRESSURE_TIMEOUT_MS) + } + + const send = (event: WorkspaceEventPayload, cursor?: string) => { deps.logger.debug({ clientId, type: event.type }, "SSE event dispatched") if (deps.logger.isLevelEnabled("trace")) { deps.logger.trace({ clientId, event }, "SSE event payload") } - reply.raw.write(`data: ${JSON.stringify(event)}\n\n`) + const frame = `${cursor === undefined ? "" : `id: ${cursor}\n`}data: ${JSON.stringify(event)}\n\n` + if (bootstrapFrames) bootstrapFrames.push(frame) + else write(frame) } - const unsubscribe = deps.eventBus.onEvent(send) - const heartbeat = setInterval(() => { - const ping = { ts: Date.now() } - reply.raw.write(`event: codenomad.client.ping\ndata: ${JSON.stringify(ping)}\n\n`) - }, 15000) + const sendReplayGap = (gap: EventReplayGap) => { + deps.logger.debug({ clientId, ...gap }, "SSE replay window missed") + write(`event: codenomad.replay.reset\nid: ${gap.latestCursor}\ndata: ${JSON.stringify(gap)}\n\n`) + } - let closed = false - const close = () => { - if (closed) return - closed = true - clearInterval(heartbeat) + unsubscribe = deps.eventBus.onEvent(send, lastEventCursor, sendReplayGap) + if (bootstrapFrames) { + bootstrapFrames.push(`event: codenomad.replay.cursor\nid: ${deps.eventBus.latestCursor}\ndata: {}\n\n`) + const bootstrap = bootstrapFrames.join("") + bootstrapFrames = undefined + write(bootstrap) + } + if (closed) { unsubscribe() - reply.raw.end?.() - deps.logger.debug({ clientId }, "SSE client disconnected") + return } + heartbeat = setInterval(() => { + const ping = { ts: Date.now() } + write(`event: codenomad.client.ping\ndata: ${JSON.stringify(ping)}\n\n`) + }, 15000) const unregister = deps.registerClient(close) const unregisterConnection = deps.connectionManager.register({ @@ -87,3 +127,14 @@ export function registerEventRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(204).send() }) } + +function copyReplyHeadersToRaw(reply: FastifyReply): void { + for (const [name, value] of Object.entries(reply.getHeaders())) { + if (value !== undefined) reply.raw.setHeader(name, value) + } +} + +function readLastEventCursor(value: string | string[] | undefined): string | undefined { + const raw = Array.isArray(value) ? value[0] : value + return raw && raw.length <= 512 ? raw : undefined +} diff --git a/packages/server/src/server/routes/plugin.test.ts b/packages/server/src/server/routes/plugin.test.ts new file mode 100644 index 000000000..684d326b8 --- /dev/null +++ b/packages/server/src/server/routes/plugin.test.ts @@ -0,0 +1,43 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import Fastify from "fastify" + +import { EventBus } from "../../events/bus" +import { registerPluginRoutes } from "./plugin" + +test("plugin event callbacks require the capability on the canonical handler", async () => { + const app = Fastify({ logger: false }) + registerPluginRoutes(app, { + workspaceManager: { + get: (id: string) => id === "workspace" ? { id, status: "ready" } : undefined, + getPluginCallbackAuthorizationHeader: (id: string) => id === "workspace" ? "Bearer callback-secret" : undefined, + } as never, + eventBus: new EventBus(), + logger: { debug: () => undefined } as never, + channel: {} as never, + voiceModeManager: {} as never, + }) + + const missing = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + payload: { type: "test.event" }, + }) + const authorized = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin/event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + const doubledSlash = await app.inject({ + method: "POST", + url: "/workspaces/workspace/plugin//event", + headers: { authorization: "Bearer callback-secret" }, + payload: { type: "test.event" }, + }) + + assert.equal(missing.statusCode, 401) + assert.equal(authorized.statusCode, 204) + assert.equal(doubledSlash.statusCode, 404) + await app.close() +}) diff --git a/packages/server/src/server/routes/plugin.ts b/packages/server/src/server/routes/plugin.ts index aef570072..c8eca6ad9 100644 --- a/packages/server/src/server/routes/plugin.ts +++ b/packages/server/src/server/routes/plugin.ts @@ -1,4 +1,4 @@ -import { FastifyInstance } from "fastify" +import { FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify" import { z } from "zod" import type { VoiceModeStateResponse } from "../../api-types" import type { WorkspaceManager } from "../../workspaces/manager" @@ -7,6 +7,7 @@ import type { Logger } from "../../logger" import { PluginChannelManager } from "../../plugins/channel" import { buildPingEvent, handlePluginEvent } from "../../plugins/handlers" import { VoiceModeManager } from "../../plugins/voice-mode" +import { sendUnauthorized } from "../../auth/http-auth" interface RouteDeps { workspaceManager: WorkspaceManager @@ -29,15 +30,24 @@ const VoiceModeStateSchema = z.object({ export function registerPluginRoutes(app: FastifyInstance, deps: RouteDeps) { app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/events", (request, reply) => { + if (!isCanonicalPluginPath(request, request.params.id, "events")) { + reply.code(404).send({ error: "Unknown plugin endpoint" }) + return + } + if (!hasPluginCapability(request, request.params.id, deps)) { + sendUnauthorized(request, reply) + return + } const workspace = deps.workspaceManager.get(request.params.id) if (!workspace) { reply.code(404).send({ error: "Workspace not found" }) return } - reply.raw.setHeader("Content-Type", "text/event-stream") - reply.raw.setHeader("Cache-Control", "no-cache") - reply.raw.setHeader("Connection", "keep-alive") + reply.header("Content-Type", "text/event-stream") + reply.header("Cache-Control", "no-cache") + reply.header("Connection", "keep-alive") + copyReplyHeadersToRaw(reply) reply.raw.flushHeaders?.() reply.hijack() @@ -80,27 +90,46 @@ export function registerPluginRoutes(app: FastifyInstance, deps: RouteDeps) { return { enabled: payload.enabled } }) - const handleWildcard = async (request: any, reply: any) => { - const workspaceId = request.params.id as string + app.post<{ Params: { id: string } }>("/workspaces/:id/plugin/event", async (request, reply) => { + const workspaceId = request.params.id + if (!isCanonicalPluginPath(request, workspaceId, "event")) { + reply.code(404).send({ error: "Unknown plugin endpoint" }) + return + } + if (!hasPluginCapability(request, workspaceId, deps)) { + sendUnauthorized(request, reply) + return + } const workspace = deps.workspaceManager.get(workspaceId) if (!workspace) { reply.code(404).send({ error: "Workspace not found" }) return } - const suffix = (request.params["*"] as string | undefined) ?? "" - const normalized = suffix.replace(/^\/+/, "") + const parsed = PluginEventSchema.parse(request.body ?? {}) + handlePluginEvent(workspaceId, parsed, { workspaceManager: deps.workspaceManager, eventBus: deps.eventBus, logger: deps.logger }) + reply.code(204).send() + }) - if (normalized === "event" && request.method === "POST") { - const parsed = PluginEventSchema.parse(request.body ?? {}) - handlePluginEvent(workspaceId, parsed, { workspaceManager: deps.workspaceManager, eventBus: deps.eventBus, logger: deps.logger }) - reply.code(204).send() - return - } + app.all("/workspaces/:id/plugin/*", (_request, reply) => reply.code(404).send({ error: "Unknown plugin endpoint" })) + app.all("/workspaces/:id/plugin", (_request, reply) => reply.code(404).send({ error: "Unknown plugin endpoint" })) +} - reply.code(404).send({ error: "Unknown plugin endpoint" }) - } +function isCanonicalPluginPath(request: FastifyRequest, workspaceId: string, endpoint: string): boolean { + const pathname = (request.raw.url ?? request.url).split("?")[0] + return pathname === `/workspaces/${encodeURIComponent(workspaceId)}/plugin/${endpoint}` +} + +function hasPluginCapability(request: FastifyRequest, workspaceId: string, deps: RouteDeps): boolean { + const provided = Array.isArray(request.headers.authorization) + ? request.headers.authorization[0] + : request.headers.authorization + const expected = deps.workspaceManager.getPluginCallbackAuthorizationHeader(workspaceId) + return Boolean(expected && provided === expected) +} - app.all("/workspaces/:id/plugin/*", handleWildcard) - app.all("/workspaces/:id/plugin", handleWildcard) +function copyReplyHeadersToRaw(reply: FastifyReply): void { + for (const [name, value] of Object.entries(reply.getHeaders())) { + if (value !== undefined) reply.raw.setHeader(name, value) + } } diff --git a/packages/server/src/server/routes/workflows.test.ts b/packages/server/src/server/routes/workflows.test.ts new file mode 100644 index 000000000..1fe323c98 --- /dev/null +++ b/packages/server/src/server/routes/workflows.test.ts @@ -0,0 +1,308 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import Fastify from "fastify" +import { WORKFLOW_DEFINITION_REVISION_LIMIT } from "../../workflows/definition-schema" +import { WorkflowRunError, type WorkflowManager } from "../../workflows/manager" +import { registerWorkflowRoutes } from "./workflows" + +describe("workflow routes", () => { + it("rejects generic plugin starts and scopes plugin requests to their workspace", async () => { + const calls: unknown[] = [] + const workflowManager = { + start: async (input: unknown) => { + calls.push(input) + return { id: "00000000-0000-4000-8000-000000000001", workspaceId: "workspace-a", status: "running" } + }, + list: async () => [], + get: async () => ({ id: "run", workspaceId: "workspace-b" }), + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const rejected = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { + objective: "Ship it", + stages: [{ id: "build", title: "Build", instructions: "Implement" }], + }, + }) + assert.equal(rejected.statusCode, 404) + assert.equal(calls.length, 0) + + const foreign = await app.inject({ + method: "GET", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001", + }) + assert.equal(foreign.statusCode, 404) + + const pluginApproval = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/approve", + }) + assert.equal(pluginApproval.statusCode, 404) + await app.close() + }) + + it("exposes definition CRUD to host routes and only read/start to a workspace plugin", async () => { + const calls: Array<[string, ...unknown[]]> = [] + const record = { id: "deploy", revision: 2, definition: { id: "deploy", name: "Deploy" } } + const workflowManager = { + listDefinitions: async () => [record], + getDefinition: async (id: string, revision?: number) => { calls.push(["get", id, revision]); return record }, + createDefinition: async (source: unknown) => { calls.push(["create", source]); return record }, + updateDefinition: async (id: string, revision: number, source: unknown) => { calls.push(["update", id, revision, source]); return record }, + deleteDefinition: async (id: string, revision: number) => { calls.push(["delete", id, revision]); return true }, + validateDefinition: () => ({ valid: true }), + start: async (input: unknown) => { calls.push(["start", input]); return { id: "run", workspaceId: "workspace-a" } }, + startLatest: async (input: unknown) => { calls.push(["startLatest", input]); return { id: "run", workspaceId: "workspace-a" } }, + pause: async (id: string) => { calls.push(["pause", id]); return { id } }, + resume: async (id: string, confirm: boolean) => { calls.push(["resume", id, confirm]); return { id } }, + answer: async (id: string, executionNodeId: string, answer: unknown) => { calls.push(["answer", id, executionNodeId, answer]); return { id } }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + assert.equal((await app.inject({ method: "GET", url: "/workspaces/workspace-a/plugin/workflow-definitions" })).statusCode, 200) + assert.equal((await app.inject({ method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions", payload: { + source: "version: 1", + } })).statusCode, 404) + assert.equal((await app.inject({ method: "PUT", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy", payload: { + expectedRevision: 2, definition: { version: 1 }, + } })).statusCode, 404) + const started = await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { runId: "00000000-0000-4000-8000-000000000001", objective: "Release", inputs: { environment: "test" } }, + }) + assert.equal(started.statusCode, 202) + assert.deepEqual(calls.at(-1), ["start", { + workspaceId: "workspace-a", definitionId: "deploy", runId: "00000000-0000-4000-8000-000000000001", objective: "Release", + inputs: { environment: "test" }, + }]) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { runId: "not-a-uuid" }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { + initiatorSessionId: "parent-session", + objective: "Ship it", + stages: [{ id: "build", title: "Build", instructions: "Implement" }], + }, + })).statusCode, 404) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { worktree: { mode: "existing", slug: "review" } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { initiatorSessionId: "victim-session" }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace-a", worktree: { mode: "current", slug: "not-allowed" } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy/start", + payload: { definitionRevision: 1 }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-runs", + payload: { definitionId: "deploy" }, + })).statusCode, 404) + assert.equal((await app.inject({ + method: "DELETE", url: "/workspaces/workspace-a/plugin/workflow-definitions/deploy?expectedRevision=2", + })).statusCode, 404) + assert.equal((await app.inject({ + method: "POST", url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/answer", + payload: { answer: true }, + })).statusCode, 404) + + assert.equal((await app.inject({ + method: "PUT", url: "/api/workflow-definitions/deploy", + payload: { expectedRevision: 2, definition: { version: 1 } }, + })).statusCode, 200) + assert.deepEqual(calls.at(-1), ["update", "deploy", 2, { version: 1 }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace-a", definitionRevision: 2, inputs: { environment: "prod" } }, + })).statusCode, 202) + assert.deepEqual(calls.at(-1), ["startLatest", { + workspaceId: "workspace-a", definitionId: "deploy", definitionRevision: 2, inputs: { environment: "prod" }, + }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs/00000000-0000-4000-8000-000000000001/answer", + payload: { executionNodeId: "00000000-0000-4000-8000-000000000002", answer: { approved: true } }, + })).statusCode, 200) + assert.deepEqual(calls.at(-1), ["answer", "00000000-0000-4000-8000-000000000001", "00000000-0000-4000-8000-000000000002", { approved: true }]) + await app.close() + }) + + it("keeps legacy and definition starts exclusive and enforces atomic latest revisions", async () => { + const calls: Array<[string, unknown]> = [] + const workflowManager = { + getDefinition: async () => ({ id: "deploy", revision: 2, definition: { id: "deploy" } }), + start: async (input: unknown) => { calls.push(["start", input]); return { id: "legacy" } }, + startLatest: async (input: { definitionRevision?: number }) => { + if (input.definitionRevision !== undefined && input.definitionRevision !== 2) { + throw new WorkflowRunError("Workflow definition revision is stale", 409) + } + calls.push(["startLatest", input]) + return { id: "saved" } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const legacy = { + workspaceId: "workspace", objective: "Ship", + stages: [{ id: "build", title: "Build", instructions: "Build it" }], + } + assert.equal((await app.inject({ method: "POST", url: "/api/workflow-runs", payload: legacy })).statusCode, 202) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { ...legacy, definitionId: "deploy" }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { + ...legacy, stages: [{ ...legacy.stages[0], definitionId: "deploy" }], + }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-runs", payload: { + workspaceId: "workspace", definitionId: "deploy", stages: legacy.stages, + }, + })).statusCode, 400) + + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 1 }, + })).statusCode, 409) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 2 }, + })).statusCode, 202) + assert.deepEqual(calls.at(-1), ["startLatest", { + workspaceId: "workspace", definitionId: "deploy", definitionRevision: 2, + }]) + assert.equal((await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: WORKFLOW_DEFINITION_REVISION_LIMIT + 1 }, + })).statusCode, 400) + await app.close() + }) + + it("fails closed when atomic latest start manager integration is unavailable", async () => { + const workflowManager = { + getDefinition: async () => ({ id: "deploy", revision: 2, definition: { id: "deploy" } }), + start: async () => { throw new Error("non-atomic start must not be called") }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const response = await app.inject({ + method: "POST", url: "/api/workflow-definitions/deploy/start", + payload: { workspaceId: "workspace", definitionRevision: 2 }, + }) + assert.equal(response.statusCode, 501) + await app.close() + }) + + it("bounds definition inputs and gate answers without recursive validation", async () => { + const calls: Array<[string, ...unknown[]]> = [] + const workflowManager = { + start: async (input: unknown) => { calls.push(["start", input]); return { id: "run" } }, + answer: async (...args: unknown[]) => { calls.push(["answer", ...args]); return { id: "run" } }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const startUrl = "/api/workflow-definitions/deploy/start" + const pluginStartUrl = "/workspaces/workspace/plugin/workflow-definitions/deploy/start" + const answerUrl = "/api/workflow-runs/00000000-0000-4000-8000-000000000001/answer" + const executionNodeId = "00000000-0000-4000-8000-000000000002" + + let nested: unknown = true + for (let depth = 0; depth < 21; depth++) nested = { nested } + assert.equal((await app.inject({ + method: "POST", url: pluginStartUrl, payload: { inputs: nested }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: startUrl, + payload: { workspaceId: "workspace", inputs: { values: Array(50_001).fill(null) } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: startUrl, + payload: { workspaceId: "workspace", inputs: { value: "é".repeat(128_001) } }, + })).statusCode, 400) + assert.equal((await app.inject({ + method: "POST", url: answerUrl, payload: { executionNodeId, answer: nested }, + })).statusCode, 400) + assert.equal(calls.length, 0) + + assert.equal((await app.inject({ + method: "POST", url: answerUrl, payload: { executionNodeId, answer: { approved: true } }, + })).statusCode, 200) + assert.deepEqual(calls, [["answer", "00000000-0000-4000-8000-000000000001", executionNodeId, { approved: true }]]) + await app.close() + }) + + it("requires the expected legacy stage when approving", async () => { + const calls: string[] = [] + const workflowManager = { + approve: async (id: string, expectedStepId: string) => { + if (expectedStepId !== "review-stage") throw new WorkflowRunError("Workflow approval is stale", 409) + calls.push(id) + return { id } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const url = "/api/workflow-runs/00000000-0000-4000-8000-000000000001/approve" + + assert.equal((await app.inject({ method: "POST", url })).statusCode, 400) + assert.equal((await app.inject({ method: "POST", url, payload: { expectedStepId: "stale-stage" } })).statusCode, 409) + assert.equal(calls.length, 0) + assert.equal((await app.inject({ method: "POST", url, payload: { expectedStepId: "review-stage" } })).statusCode, 200) + assert.deepEqual(calls, ["00000000-0000-4000-8000-000000000001"]) + await app.close() + }) + + it("requires a revision-bound recovery confirmation", async () => { + const calls: unknown[][] = [] + const workflowManager = { + resume: async (...args: unknown[]) => { calls.push(args); return { id: args[0] } }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + const url = "/api/workflow-runs/00000000-0000-4000-8000-000000000001/resume" + + assert.equal((await app.inject({ method: "POST", url, payload: { confirmRecovery: true } })).statusCode, 400) + assert.equal((await app.inject({ method: "POST", url, payload: { expectedRevision: 4 } })).statusCode, 400) + assert.equal((await app.inject({ method: "POST", url, payload: { + confirmRecovery: true, expectedRevision: 4, + } })).statusCode, 200) + assert.deepEqual(calls, [["00000000-0000-4000-8000-000000000001", true, 4]]) + await app.close() + }) + + it("cancels plugin-owned runs without joining get", async () => { + const calls: unknown[][] = [] + const workflowManager = { + get: async () => { throw new Error("get must not be called") }, + cancelOwned: async (...args: unknown[]) => { + calls.push(args) + return { id: args[0], workspaceId: args[1], status: "cancelled" } + }, + } as unknown as WorkflowManager + const app = Fastify({ logger: false }) + registerWorkflowRoutes(app, { workflowManager }) + + const response = await app.inject({ + method: "POST", + url: "/workspaces/workspace-a/plugin/workflow-runs/00000000-0000-4000-8000-000000000001/cancel", + }) + assert.equal(response.statusCode, 200) + assert.deepEqual(calls, [["00000000-0000-4000-8000-000000000001", "workspace-a"]]) + await app.close() + }) +}) diff --git a/packages/server/src/server/routes/workflows.ts b/packages/server/src/server/routes/workflows.ts new file mode 100644 index 000000000..f0a2d491c --- /dev/null +++ b/packages/server/src/server/routes/workflows.ts @@ -0,0 +1,502 @@ +import type { FastifyInstance } from "fastify" +import { z } from "zod" +import type { WorkflowDefinitionRunCreateRequest, WorkflowRun } from "../../api-types" +import { WORKFLOW_DEFINITION_REVISION_LIMIT, WORKFLOW_LIMITS } from "../../workflows/definition-schema" +import { WorkflowDefinitionStoreError } from "../../workflows/definition-store" +import type { WorkflowManager } from "../../workflows/manager" +import { WorkflowRunError } from "../../workflows/manager" + +type WorkflowManagerWithLatestStart = WorkflowManager & { + startLatest?: (input: WorkflowDefinitionRunCreateRequest) => Promise +} + +interface RouteDeps { + workflowManager: WorkflowManagerWithLatestStart +} + +const DefinitionRevisionSchema = z.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT) + +const ModelSchema = z.object({ + providerID: z.string().trim().min(1).max(200), + modelID: z.string().trim().min(1).max(200), +}).strict() + +const StageSchema = z.object({ + id: z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + title: z.string().trim().min(1).max(200), + instructions: z.string().trim().min(1).max(20_000), + agent: z.string().trim().min(1).max(200).optional(), + model: ModelSchema.optional(), + requiresApproval: z.boolean().optional(), +}).strict() + +const CreateObjectSchema = z.object({ + workspaceId: z.string().trim().min(1).max(200), + initiatorSessionId: z.string().trim().min(1).max(200).optional(), + objective: z.string().trim().min(1).max(50_000), + stages: z.array(StageSchema).min(1).max(12), +}).strict() + +const WorktreePolicySchema = z.discriminatedUnion("mode", [ + z.object({ mode: z.literal("current") }).strict(), + z.object({ mode: z.literal("existing"), slug: z.string().trim().min(1).max(200) }).strict(), + z.object({ mode: z.literal("new"), slug: z.string().trim().min(1).max(200) }).strict(), +]) + +const DefinitionStartObjectSchema = z.object({ + workspaceId: z.string().trim().min(1).max(200), + initiatorSessionId: z.string().trim().min(1).max(200).optional(), + objective: z.string().trim().min(1).max(50_000).optional(), + definitionId: z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/), + definitionRevision: DefinitionRevisionSchema.optional(), + inputs: z.record(z.unknown()).superRefine(validateBoundedJson).optional(), + worktree: WorktreePolicySchema.optional(), +}).strict() + +const requireUniqueStageIds = (value: { stages: Array<{ id: string }> }, ctx: z.RefinementCtx) => { + const ids = new Set() + value.stages.forEach((stage, index) => { + if (ids.has(stage.id)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Stage IDs must be unique", path: ["stages", index, "id"] }) + } + ids.add(stage.id) + }) +} + +const LegacyCreateSchema = CreateObjectSchema.superRefine(requireUniqueStageIds) +const CreateSchema = z.union([LegacyCreateSchema, DefinitionStartObjectSchema]) + +const RunIdSchema = z.string().uuid() +const ListSchema = z.object({ workspaceId: z.string().trim().min(1).max(200).optional() }) +const PluginDefinitionStartSchema = z.object({ + runId: z.string().uuid().optional(), + objective: z.string().trim().min(1).max(50_000).optional(), + inputs: z.record(z.unknown()).superRefine(validateBoundedJson).optional(), +}).strict() +const DefinitionIdSchema = z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/) +const DefinitionRevisionQuerySchema = z.object({ + revision: z.coerce.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT).optional(), +}) +const DefinitionSourceSchema = z.object({ + source: z.string().optional(), + definition: z.unknown().optional(), +}).strict().superRefine((value, ctx) => { + const supplied = Number(value.source !== undefined) + Number(Object.prototype.hasOwnProperty.call(value, "definition")) + if (supplied !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Supply exactly one of source or definition" }) +}) +const DefinitionUpdateSchema = z.object({ + expectedRevision: DefinitionRevisionSchema, + source: z.string().optional(), + definition: z.unknown().optional(), +}).strict().superRefine((value, ctx) => { + const supplied = Number(value.source !== undefined) + Number(Object.prototype.hasOwnProperty.call(value, "definition")) + if (supplied !== 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Supply exactly one of source or definition" }) +}) +const DeleteDefinitionQuerySchema = z.object({ + expectedRevision: z.coerce.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT), +}) +const ResumeSchema = z.object({ + confirmRecovery: z.boolean().optional(), + expectedRevision: z.number().int().min(0).optional(), +}).strict().superRefine((value, ctx) => { + if (value.confirmRecovery === true && value.expectedRevision === undefined) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "expectedRevision is required for recovery" }) + } else if (value.confirmRecovery !== true && value.expectedRevision !== undefined) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "expectedRevision requires recovery confirmation" }) + } +}) +const ApprovalSchema = z.object({ expectedStepId: z.string().trim().min(1).max(100) }).strict() +const AnswerSchema = z.object({ executionNodeId: z.string().uuid(), answer: z.unknown() }).strict().superRefine((value, ctx) => { + if (!Object.prototype.hasOwnProperty.call(value, "answer")) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "answer is required" }) + else validateBoundedJson(value.answer, ctx) +}) + +const JSON_VALUE_COUNT_LIMIT = 50_000 + +function validateBoundedJson(value: unknown, ctx: z.RefinementCtx): void { + const issue = inspectBoundedJson(value) + if (issue) ctx.addIssue({ code: z.ZodIssueCode.custom, message: issue }) +} + +function inspectBoundedJson(input: unknown): string | undefined { + const pending: Array<{ value: unknown; depth: number }> = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let count = 0 + let bytes = 0 + + while (pending.length) { + const { value, depth } = pending.pop()! + if (++count > JSON_VALUE_COUNT_LIMIT) return "JSON value contains too many values" + if (depth > WORKFLOW_LIMITS.valueDepth) return "JSON value is too deeply nested" + if (value === null) bytes += 4 + else if (typeof value === "string") bytes += Buffer.byteLength(JSON.stringify(value), "utf8") + else if (typeof value === "boolean") bytes += value ? 4 : 5 + else if (typeof value === "number" && Number.isFinite(value)) bytes += JSON.stringify(value).length + else if (value && typeof value === "object") { + if (seen.has(value)) return "JSON value must not contain cycles or aliases" + seen.add(value) + if (Array.isArray(value)) { + const keys = Object.keys(value) + if (keys.length !== value.length || keys.some((key, index) => key !== String(index))) { + return "JSON value must contain only plain arrays" + } + bytes += 2 + Math.max(0, value.length - 1) + for (let index = value.length - 1; index >= 0; index--) { + pending.push({ value: value[index], depth: depth + 1 }) + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return "JSON value must contain only plain objects" + const keys = Object.keys(value) + if (Reflect.ownKeys(value).length !== keys.length) return "JSON value must contain only plain objects" + bytes += 2 + Math.max(0, keys.length - 1) + for (const key of keys) { + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (!descriptor || !("value" in descriptor) || !descriptor.enumerable) return "JSON value must contain only plain objects" + bytes += Buffer.byteLength(JSON.stringify(key), "utf8") + 1 + pending.push({ value: descriptor.value, depth: depth + 1 }) + } + } + } else return "JSON value must contain only JSON values" + + if (bytes > WORKFLOW_LIMITS.sourceBytes) return "JSON value is too large" + } +} + +const definitionSource = (value: { source?: string; definition?: unknown }) => value.source ?? value.definition +const belongsToWorkspace = (run: { workspaceId: string; worktreeSelection?: { sourceWorkspaceId: string } }, workspaceId: string) => + run.workspaceId === workspaceId || run.worktreeSelection?.sourceWorkspaceId === workspaceId + +const handleWorkflowError = (error: unknown, reply: { code(statusCode: number): unknown }) => { + if (error instanceof WorkflowRunError || error instanceof WorkflowDefinitionStoreError) { + reply.code(error.statusCode) + return { error: error.message } + } + if (error instanceof z.ZodError) { + reply.code(400) + return { error: "Invalid workflow definition", issues: error.flatten() } + } + throw error +} + +async function startLatestDefinition( + manager: WorkflowManagerWithLatestStart, + input: WorkflowDefinitionRunCreateRequest, +): Promise { + if (!manager.startLatest) { + throw new WorkflowRunError("Atomic saved workflow start is unavailable", 501) + } + return manager.startLatest(input) +} + +export function registerWorkflowRoutes(app: FastifyInstance, deps: RouteDeps) { + app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/workflow-definitions", async () => { + return { definitions: await deps.workflowManager.listDefinitions() } + }) + + app.get<{ Params: { id: string; definitionId: string } }>( + "/workspaces/:id/plugin/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + if (!id.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + const definition = await deps.workflowManager.getDefinition(id.data) + if (!definition) { + reply.code(404) + return { error: "Workflow definition not found" } + } + return definition + }, + ) + + app.post<{ Params: { id: string; definitionId: string } }>( + "/workspaces/:id/plugin/workflow-definitions/:definitionId/start", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = PluginDefinitionStartSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow request" } + } + try { + const run = await deps.workflowManager.start({ + ...body.data, + workspaceId: request.params.id, + definitionId: id.data, + }) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.get<{ Params: { id: string } }>("/workspaces/:id/plugin/workflow-runs", async (request) => { + return { runs: await deps.workflowManager.list(request.params.id) } + }) + + app.get<{ Params: { id: string; runId: string } }>( + "/workspaces/:id/plugin/workflow-runs/:runId", + async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + const run = await deps.workflowManager.get(parsed.data, request.params.id) + if (!run || !belongsToWorkspace(run, request.params.id)) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + }, + ) + + app.post<{ Params: { id: string; runId: string } }>( + "/workspaces/:id/plugin/workflow-runs/:runId/cancel", + async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.cancelOwned(parsed.data, request.params.id) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.post("/api/workflow-definitions/validate", async (request, reply) => { + const parsed = DefinitionSourceSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { valid: false, issues: parsed.error.issues } + } + const result = deps.workflowManager.validateDefinition(definitionSource(parsed.data)) + if (!result.valid) reply.code(400) + return result + }) + + app.get("/api/workflow-definitions", async () => ({ definitions: await deps.workflowManager.listDefinitions() })) + + app.post("/api/workflow-definitions", async (request, reply) => { + const parsed = DefinitionSourceSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow definition request", issues: parsed.error.flatten() } + } + try { + const record = await deps.workflowManager.createDefinition(definitionSource(parsed.data)) + reply.code(201) + return record + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get<{ Params: { definitionId: string }; Querystring: { revision?: string } }>( + "/api/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const query = DefinitionRevisionQuerySchema.safeParse(request.query) + if (!id.success || !query.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + const definition = await deps.workflowManager.getDefinition(id.data, query.data.revision) + if (!definition) { + reply.code(404) + return { error: "Workflow definition not found" } + } + return definition + }, + ) + + app.put<{ Params: { definitionId: string } }>("/api/workflow-definitions/:definitionId", async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = DefinitionUpdateSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + try { + return await deps.workflowManager.updateDefinition(id.data, body.data.expectedRevision, definitionSource(body.data)) + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.delete<{ Params: { definitionId: string }; Querystring: { expectedRevision?: string } }>( + "/api/workflow-definitions/:definitionId", + async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const query = DeleteDefinitionQuerySchema.safeParse(request.query) + if (!id.success || !query.success) { + reply.code(400) + return { error: "Invalid workflow definition request" } + } + try { + const deleted = await deps.workflowManager.deleteDefinition(id.data, query.data.expectedRevision) + if (!deleted) { + reply.code(404) + return { error: "Workflow definition not found" } + } + reply.code(204) + return undefined + } catch (error) { + return handleWorkflowError(error, reply) + } + }, + ) + + app.post<{ Params: { definitionId: string } }>("/api/workflow-definitions/:definitionId/start", async (request, reply) => { + const id = DefinitionIdSchema.safeParse(request.params.definitionId) + const body = DefinitionStartObjectSchema.omit({ definitionId: true }).safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow request" } + } + try { + const run = await startLatestDefinition(deps.workflowManager, { ...body.data, definitionId: id.data }) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get("/api/workflow-runs", async (request, reply) => { + const parsed = ListSchema.safeParse(request.query) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow query" } + } + return { runs: await deps.workflowManager.list(parsed.data.workspaceId) } + }) + + app.post("/api/workflow-runs", async (request, reply) => { + const parsed = CreateSchema.safeParse(request.body) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow request", issues: parsed.error.flatten() } + } + + try { + const run = "stages" in parsed.data + ? await deps.workflowManager.start(parsed.data) + : await startLatestDefinition(deps.workflowManager, parsed.data) + reply.code(202) + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.get<{ Params: { runId: string } }>("/api/workflow-runs/:runId", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + const run = await deps.workflowManager.get(parsed.data) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/cancel", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.cancel(parsed.data) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/approve", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = ApprovalSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow approval request" } + } + try { + const run = await deps.workflowManager.approve(id.data, body.data.expectedStepId) + if (!run) { + reply.code(404) + return { error: "Workflow run not found" } + } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/pause", async (request, reply) => { + const parsed = RunIdSchema.safeParse(request.params.runId) + if (!parsed.success) { + reply.code(400) + return { error: "Invalid workflow run ID" } + } + try { + const run = await deps.workflowManager.pause(parsed.data) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/resume", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = ResumeSchema.safeParse(request.body ?? {}) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow resume request" } + } + try { + const run = await deps.workflowManager.resume(id.data, body.data.confirmRecovery, body.data.expectedRevision) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) + + app.post<{ Params: { runId: string } }>("/api/workflow-runs/:runId/answer", async (request, reply) => { + const id = RunIdSchema.safeParse(request.params.runId) + const body = AnswerSchema.safeParse(request.body) + if (!id.success || !body.success) { + reply.code(400) + return { error: "Invalid workflow gate answer" } + } + try { + const run = await deps.workflowManager.answer(id.data, body.data.executionNodeId, body.data.answer) + if (!run) { reply.code(404); return { error: "Workflow run not found" } } + return run + } catch (error) { + return handleWorkflowError(error, reply) + } + }) +} diff --git a/packages/server/src/server/routes/workspaces.test.ts b/packages/server/src/server/routes/workspaces.test.ts index e115c5b7a..641ef6cd0 100644 --- a/packages/server/src/server/routes/workspaces.test.ts +++ b/packages/server/src/server/routes/workspaces.test.ts @@ -3,7 +3,7 @@ import { describe, it } from "node:test" import Fastify from "fastify" import type { WorkspaceDescriptor } from "../../api-types" -import type { WorkspaceManager } from "../../workspaces/manager" +import { WorkspaceDeletionBlockedError, WorkspacePathOwnedError, type WorkspaceManager } from "../../workspaces/manager" import { registerWorkspaceRoutes } from "./workspaces" describe("workspace routes", () => { @@ -40,6 +40,7 @@ describe("workspace routes", () => { path: "C:/work", name: "Work", binaryPath: " C:/tools/opencode.exe ", + lineageId: "00000000-0000-4000-8000-000000000001", requestId: " restore-request ", forceNew: true, }, @@ -48,6 +49,7 @@ describe("workspace routes", () => { assert.equal(response.statusCode, 201) assert.deepEqual(calls, [["C:/work", "Work", { binaryPath: "C:/tools/opencode.exe", + lineageId: "00000000-0000-4000-8000-000000000001", requestId: "restore-request", forceNew: true, }]]) @@ -123,4 +125,44 @@ describe("workspace routes", () => { assert.equal((await cancellation).statusCode, 204) await app.close() }) + + it("reports workflow-owned deletion and restore cancellation as conflicts", async () => { + let deleted = false + const app = Fastify({ logger: false }) + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/worktree" }), + delete: async () => { + deleted = true + throw new WorkspaceDeletionBlockedError("workspace") + }, + cancelCreationRequest: async () => { throw new WorkspaceDeletionBlockedError("workspace") }, + } as unknown as WorkspaceManager + registerWorkspaceRoutes(app, { workspaceManager }) + + const response = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace" }) + const cancellation = await app.inject({ + method: "POST", + url: "/api/workspaces/creation/cancel", + payload: { requestId: "restore-request" }, + }) + + assert.equal(response.statusCode, 409) + assert.equal(cancellation.statusCode, 409) + assert.equal(deleted, true) + await app.close() + }) + + it("reports a workspace path owned by another server as a conflict", async () => { + const app = Fastify({ logger: false }) + registerWorkspaceRoutes(app, { + workspaceManager: { + create: async () => { throw new WorkspacePathOwnedError("C:/worktree") }, + } as unknown as WorkspaceManager, + }) + + const response = await app.inject({ method: "POST", url: "/api/workspaces", payload: { path: "C:/worktree" } }) + assert.equal(response.statusCode, 409) + assert.match(response.body, /another CodeNomad server/) + await app.close() + }) }) diff --git a/packages/server/src/server/routes/workspaces.ts b/packages/server/src/server/routes/workspaces.ts index e7f052136..ea81bfccd 100644 --- a/packages/server/src/server/routes/workspaces.ts +++ b/packages/server/src/server/routes/workspaces.ts @@ -1,6 +1,6 @@ import { FastifyInstance, FastifyReply } from "fastify" import { z } from "zod" -import { WorkspaceManager } from "../../workspaces/manager" +import { WorkspaceDeletionBlockedError, WorkspaceManager, WorkspacePathOwnedError } from "../../workspaces/manager" import { getWorktreeGitDiff, getWorktreeGitStatus } from "../../workspaces/git-status" import { commitWorktreeChanges, isGitMutationError, stageWorktreePaths, unstageWorktreePaths } from "../../workspaces/git-mutations" import { cloneGitRepository, isGitCloneError } from "../../workspaces/git-clone" @@ -13,6 +13,7 @@ interface RouteDeps { const WorkspaceCreateSchema = z.object({ path: z.string(), + lineageId: z.string().uuid().optional(), name: z.string().optional(), binaryPath: z.string().trim().min(1).max(4096).optional(), requestId: z.string().trim().min(1).max(128).optional(), @@ -79,13 +80,14 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { binaryPath: body.binaryPath, requestId: body.requestId, forceNew: body.forceNew, + ...(body.lineageId ? { lineageId: body.lineageId } : {}), }) reply.code(201) return result.created ? result.workspace : { ...result.workspace, reused: true as const } } catch (error) { request.log.error({ err: error }, "Failed to create workspace") const message = error instanceof Error ? error.message : "Failed to create workspace" - reply.code(400).type("text/plain").send(message) + reply.code(error instanceof WorkspacePathOwnedError ? 409 : 400).type("text/plain").send(message) } }) @@ -110,8 +112,12 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { }) app.delete<{ Params: { id: string } }>("/api/workspaces/:id", async (request, reply) => { - await deps.workspaceManager.delete(request.params.id) - reply.code(204) + try { + await deps.workspaceManager.delete(request.params.id) + reply.code(204) + } catch (error) { + return handleWorkspaceError(error, reply) + } }) app.post("/api/workspaces/creation/cancel", async (request, reply) => { @@ -120,8 +126,12 @@ export function registerWorkspaceRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(400).type("text/plain").send("Invalid workspace creation request") return } - await deps.workspaceManager.cancelCreationRequest(parsed.data.requestId) - reply.code(204) + try { + await deps.workspaceManager.cancelCreationRequest(parsed.data.requestId) + reply.code(204) + } catch (error) { + return handleWorkspaceError(error, reply) + } }) app.post<{ Params: { id: string } }>("/api/workspaces/:id/creation/release", async (request, reply) => { @@ -331,6 +341,10 @@ async function resolveGitWorktreeDirectory( function handleWorkspaceError(error: unknown, reply: FastifyReply) { + if (error instanceof WorkspaceDeletionBlockedError || error instanceof WorkspacePathOwnedError) { + reply.code(409) + return { error: error.message } + } if (isGitCloneError(error)) { reply.code(error.statusCode) return { error: error.message } diff --git a/packages/server/src/server/routes/worktrees.test.ts b/packages/server/src/server/routes/worktrees.test.ts new file mode 100644 index 000000000..8e1a7e421 --- /dev/null +++ b/packages/server/src/server/routes/worktrees.test.ts @@ -0,0 +1,312 @@ +import assert from "node:assert/strict" +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" +import { test } from "node:test" +import Fastify from "fastify" +import pino from "pino" + +import { EventBus } from "../../events/bus" +import { createManagedWorktree } from "../../workspaces/git-worktrees" +import { WorkspaceManager } from "../../workspaces/manager" +import { registerWorktreeRoutes } from "./worktrees" + +test("managed worktree deletion rejects workflow ownership and live workspaces", async (context) => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-worktree-route-")) + const git = (...args: string[]) => spawnSync("git", args, { cwd: temp, encoding: "utf8" }) + if (git("--version").error) { + context.skip("Git is unavailable") + rmSync(temp, { recursive: true, force: true }) + return + } + try { + assert.equal(git("init").status, 0) + assert.equal(git("config", "user.email", "test@example.com").status, 0) + assert.equal(git("config", "user.name", "CodeNomad Test").status, 0) + writeFileSync(path.join(temp, "file.txt"), "initial") + assert.equal(git("add", "file.txt").status, 0) + assert.equal(git("commit", "-m", "initial").status, 0) + const worktree = await createManagedWorktree({ repoRoot: temp, workspaceFolder: temp, slug: "review" }) + + const app = Fastify({ logger: false }) + registerWorktreeRoutes(app, { + workspaceManager: { + get: () => ({ id: "workspace", path: temp }), + withWorkspacePathLease: async (_path: string, operation: (active: boolean) => Promise) => operation(false), + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + list: async () => { throw new Error("capped list must not be used") }, + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(true), + } as never, + }) + + const response = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + + assert.equal(response.statusCode, 409) + assert.equal(existsSync(worktree.directory), true) + await app.close() + + const workspaceApp = Fastify({ logger: false }) + registerWorktreeRoutes(workspaceApp, { + workspaceManager: { + get: (id: string) => id === "workspace" + ? { id, path: temp } + : id === "execution" ? { id, path: worktree.directory, status: "ready" } : undefined, + withWorkspacePathLease: async (_path: string, operation: (active: boolean) => Promise) => operation(true), + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const activeWorkspace = await workspaceApp.inject({ + method: "DELETE", + url: "/api/workspaces/workspace/worktrees/review", + }) + assert.equal(activeWorkspace.statusCode, 409) + assert.match(activeWorkspace.body, /active workspace/) + assert.equal(existsSync(worktree.directory), true) + await workspaceApp.close() + + const terminalWorkspaceApp = Fastify({ logger: false }) + registerWorktreeRoutes(terminalWorkspaceApp, { + workspaceManager: { + get: (id: string) => id === "workspace" + ? { id, path: temp } + : id === "stopped" ? { id, path: worktree.directory, status: "stopped" } + : id === "error" ? { id, path: worktree.directory, status: "error" } : undefined, + withWorkspacePathLease: async (_path: string, operation: (active: boolean) => Promise) => operation(false), + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const terminalWorkspaces = await terminalWorkspaceApp.inject({ + method: "DELETE", + url: "/api/workspaces/workspace/worktrees/review", + }) + assert.equal(terminalWorkspaces.statusCode, 204) + assert.equal(existsSync(worktree.directory), false) + await terminalWorkspaceApp.close() + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) + +test("managed worktree POST and DELETE exclude each other on the target path", async (context) => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-worktree-mutation-route-")) + const git = (...args: string[]) => spawnSync("git", args, { cwd: temp, encoding: "utf8" }) + if (git("--version").error) { + context.skip("Git is unavailable") + rmSync(temp, { recursive: true, force: true }) + return + } + try { + assert.equal(git("init").status, 0) + assert.equal(git("config", "user.email", "test@example.com").status, 0) + assert.equal(git("config", "user.name", "CodeNomad Test").status, 0) + writeFileSync(path.join(temp, "file.txt"), "initial") + assert.equal(git("add", "file.txt").status, 0) + assert.equal(git("commit", "-m", "initial").status, 0) + const worktree = await createManagedWorktree({ repoRoot: temp, workspaceFolder: temp, slug: "review" }) + + let releaseFirst!: () => void + const firstBlocked = new Promise((resolve) => { releaseFirst = resolve }) + let firstEntered!: () => void + const firstEntry = new Promise((resolve) => { firstEntered = resolve }) + const tails = new Map>() + const leasePaths: string[] = [] + let activeOperations = 0 + let maxActiveOperations = 0 + let blockFirst = true + const workspaceManager = { + get: () => ({ id: "workspace", path: temp }), + withWorkspacePathLease: async (target: string, operation: (active: boolean) => Promise) => { + const key = process.platform === "win32" ? path.resolve(target).toLowerCase() : path.resolve(target) + leasePaths.push(key) + const previous = tails.get(key) ?? Promise.resolve() + let release!: () => void + const current = new Promise((resolve) => { release = resolve }) + tails.set(key, current) + await previous + activeOperations += 1 + maxActiveOperations = Math.max(maxActiveOperations, activeOperations) + try { + if (blockFirst) { + blockFirst = false + firstEntered() + await firstBlocked + } + return await operation(false) + } finally { + activeOperations -= 1 + release() + if (tails.get(key) === current) tails.delete(key) + } + }, + } + const app = Fastify({ logger: false }) + registerWorktreeRoutes(app, { + workspaceManager: workspaceManager as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const deletion = app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + await firstEntry + const creation = app.inject({ + method: "POST", + url: "/api/workspaces/workspace/worktrees", + payload: { slug: "review" }, + }) + while (leasePaths.length < 2) await new Promise((resolve) => setImmediate(resolve)) + assert.equal(maxActiveOperations, 1) + assert.equal(leasePaths[0], leasePaths[1]) + + releaseFirst() + const [deleted, created] = await Promise.all([deletion, creation]) + assert.equal(deleted.statusCode, 204) + assert.equal(created.statusCode, 201) + assert.equal(maxActiveOperations, 1) + assert.equal(existsSync(worktree.directory), true) + await app.close() + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) + +test("stale managed worktree DELETE does not remove a replacement", async (context) => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-stale-worktree-delete-")) + const git = (...args: string[]) => spawnSync("git", args, { cwd: temp, encoding: "utf8" }) + if (git("--version").error) { + context.skip("Git is unavailable") + rmSync(temp, { recursive: true, force: true }) + return + } + try { + assert.equal(git("init").status, 0) + assert.equal(git("config", "user.email", "test@example.com").status, 0) + assert.equal(git("config", "user.name", "CodeNomad Test").status, 0) + writeFileSync(path.join(temp, "file.txt"), "initial") + assert.equal(git("add", "file.txt").status, 0) + assert.equal(git("commit", "-m", "initial").status, 0) + const original = await createManagedWorktree({ repoRoot: temp, workspaceFolder: temp, slug: "review" }) + + let releaseStale!: () => void + let staleEntered!: () => void + const staleBlocked = new Promise((resolve) => { releaseStale = resolve }) + const staleEntry = new Promise((resolve) => { staleEntered = resolve }) + let ownershipCalls = 0 + const app = Fastify({ logger: false }) + registerWorktreeRoutes(app, { + workspaceManager: { + get: () => ({ id: "workspace", path: temp }), + withWorkspacePathLease: async (_path: string, operation: (active: boolean) => Promise) => operation(false), + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => { + ownershipCalls += 1 + if (ownershipCalls === 1) { + staleEntered() + await staleBlocked + } + return operation(false) + }, + } as never, + }) + + const staleDeletion = app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + await staleEntry + const currentDeletion = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + assert.equal(currentDeletion.statusCode, 204) + assert.equal(existsSync(original.directory), false) + + const replacement = await app.inject({ + method: "POST", + url: "/api/workspaces/workspace/worktrees", + payload: { slug: "review" }, + }) + assert.equal(replacement.statusCode, 201) + assert.equal(existsSync(original.directory), true) + + releaseStale() + const staleResponse = await staleDeletion + assert.equal(staleResponse.statusCode, 409) + assert.match(staleResponse.body, /changed while deletion was pending/) + assert.equal(existsSync(original.directory), true) + await app.close() + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) + +test("a second server cannot delete a managed worktree leased by another manager", async (context) => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-cross-server-worktree-")) + const git = (...args: string[]) => spawnSync("git", args, { cwd: temp, encoding: "utf8" }) + if (git("--version").error) { + context.skip("Git is unavailable") + rmSync(temp, { recursive: true, force: true }) + return + } + try { + assert.equal(git("init").status, 0) + assert.equal(git("config", "user.email", "test@example.com").status, 0) + assert.equal(git("config", "user.name", "CodeNomad Test").status, 0) + writeFileSync(path.join(temp, "file.txt"), "initial") + assert.equal(git("add", "file.txt").status, 0) + assert.equal(git("commit", "-m", "initial").status, 0) + const worktree = await createManagedWorktree({ repoRoot: temp, workspaceFolder: temp, slug: "review" }) + const leaseDir = path.join(temp, "leases") + const manager = () => { + const workspaceManager = new WorkspaceManager({ + rootDir: temp, + settings: { getOwner: () => ({}) } as never, + binaryResolver: { resolve: () => ({ path: process.execPath, label: "Node.js" }) } as never, + eventBus: new EventBus(), + logger: pino({ level: "silent" }), + getServerBaseUrl: () => "http://127.0.0.1:4000", + workspaceLeaseDir: leaseDir, + runtime: { + launch: async () => ({ pid: process.pid, port: 4321, exitPromise: new Promise(() => undefined), getLastOutput: () => "" }), + stop: async () => undefined, + }, + }) + ;(workspaceManager as any).waitForWorkspaceReadiness = async () => undefined + return workspaceManager + } + const owner = manager(), deletingServer = manager() + await owner.create(worktree.directory) + + const app = Fastify({ logger: false }) + registerWorktreeRoutes(app, { + workspaceManager: { + get: () => ({ id: "workspace", path: temp }), + withWorkspacePathLease: deletingServer.withWorkspacePathLease.bind(deletingServer), + } as never, + sessionMetadataPersistence: {} as never, + workflowManager: { + withWorktreeOwnershipLease: async (_source: unknown, _worktree: unknown, operation: (owned: boolean) => Promise) => operation(false), + } as never, + }) + + const blocked = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + assert.equal(blocked.statusCode, 409) + assert.equal(existsSync(worktree.directory), true) + + await owner.shutdown() + const deleted = await app.inject({ method: "DELETE", url: "/api/workspaces/workspace/worktrees/review" }) + assert.equal(deleted.statusCode, 204) + assert.equal(existsSync(worktree.directory), false) + await app.close() + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) diff --git a/packages/server/src/server/routes/worktrees.ts b/packages/server/src/server/routes/worktrees.ts index d48d9ddb1..339e55d9b 100644 --- a/packages/server/src/server/routes/worktrees.ts +++ b/packages/server/src/server/routes/worktrees.ts @@ -1,20 +1,26 @@ import type { FastifyInstance, FastifyReply } from "fastify" +import { lstat, realpath } from "node:fs/promises" +import path from "node:path" import { z } from "zod" import { WorkspaceManager } from "../../workspaces/manager" import { resolveRepoRoot, listWorktrees, isValidWorktreeSlug, + isManagedWorktree, + getManagedWorktreePath, createManagedWorktree, removeWorktree, } from "../../workspaces/git-worktrees" import type { WorktreeListResponse, WorktreeMap } from "../../api-types" import type { OpencodeYoloPersistence } from "../../permissions/opencode-yolo-metadata" import { ensureCodenomadGitExclude, readWorktreeMap, writeWorktreeMap } from "../../workspaces/worktree-map" +import type { WorkflowManager } from "../../workflows/manager" interface RouteDeps { workspaceManager: WorkspaceManager sessionMetadataPersistence: OpencodeYoloPersistence + workflowManager: WorkflowManager } const WorktreeMapSchema = z.object({ @@ -102,15 +108,20 @@ export function registerWorktreeRoutes(app: FastifyInstance, deps: RouteDeps) { await ensureCodenomadGitExclude(workspace.path, request.log).catch(() => undefined) - const created = await createManagedWorktree({ - repoRoot, - workspaceFolder: workspace.path, - slug, - logger: request.log, + return await deps.workspaceManager.withWorkspacePathLease(getManagedWorktreePath(repoRoot, slug), async (active) => { + if (active) { + reply.code(409) + return { error: "Worktree is in use by an active workspace" } + } + const created = await createManagedWorktree({ + repoRoot, + workspaceFolder: workspace.path, + slug, + logger: request.log, + }) + reply.code(201) + return created }) - - reply.code(201) - return created } catch (error) { return handleError(error, reply) } @@ -146,36 +157,68 @@ export function registerWorktreeRoutes(app: FastifyInstance, deps: RouteDeps) { reply.code(404) return { error: "Worktree not found" } } + if (!await isManagedWorktree({ repoRoot, worktree: match })) { + reply.code(404) + return { error: "Managed worktree not found" } + } + const matchIdentity = await getManagedWorktreeIdentity(match.directory) + return await deps.workflowManager.withWorktreeOwnershipLease( + { id: workspace.id, lineageId: workspace.lineageId, path: workspace.path }, + { slug, path: match.directory }, + async (owned) => { + if (owned) { + reply.code(409) + return { error: "Worktree is owned by an active workflow" } + } + return deps.workspaceManager.withWorkspacePathLease(match.directory, async (active) => { + if (active) { + reply.code(409) + return { error: "Worktree is in use by an active workspace" } + } - await removeWorktree({ workspaceFolder: workspace.path, directory: match.directory, force, logger: request.log }) + const currentWorktrees = await listWorktrees({ repoRoot, workspaceFolder: workspace.path, logger: request.log }) + const currentMatch = currentWorktrees.find((worktree) => worktree.slug === slug) + if (!currentMatch || currentMatch.kind === "root" || !await isManagedWorktree({ repoRoot, worktree: currentMatch })) { + reply.code(404) + return { error: "Managed worktree not found" } + } + if (await getManagedWorktreeIdentity(currentMatch.directory) !== matchIdentity) { + reply.code(409) + return { error: "Worktree changed while deletion was pending" } + } - // Best-effort: prune any mappings that point at the deleted worktree. - const current = await readWorktreeMap(workspace.path, request.log) - let changed = false - const nextMapping: Record = { ...(current.parentSessionWorktreeSlug ?? {}) } - for (const [sessionId, mapped] of Object.entries(nextMapping)) { - if (mapped === slug) { - delete nextMapping[sessionId] - changed = true - } - } - const nextDefault = current.defaultWorktreeSlug === slug ? "root" : current.defaultWorktreeSlug - if (nextDefault !== current.defaultWorktreeSlug) { - changed = true - } - if (changed) { - await writeWorktreeMap( - workspace.path, - { - version: 1, - defaultWorktreeSlug: nextDefault, - parentSessionWorktreeSlug: nextMapping, - }, - request.log, - ) - } + await removeWorktree({ workspaceFolder: workspace.path, directory: currentMatch.directory, force, logger: request.log }) - reply.code(204) + // Best-effort: prune any mappings that point at the deleted worktree. + const current = await readWorktreeMap(workspace.path, request.log) + let changed = false + const nextMapping: Record = { ...(current.parentSessionWorktreeSlug ?? {}) } + for (const [sessionId, mapped] of Object.entries(nextMapping)) { + if (mapped === slug) { + delete nextMapping[sessionId] + changed = true + } + } + const nextDefault = current.defaultWorktreeSlug === slug ? "root" : current.defaultWorktreeSlug + if (nextDefault !== current.defaultWorktreeSlug) { + changed = true + } + if (changed) { + await writeWorktreeMap( + workspace.path, + { + version: 1, + defaultWorktreeSlug: nextDefault, + parentSessionWorktreeSlug: nextMapping, + }, + request.log, + ) + } + + reply.code(204) + }) + } + ) } catch (error) { return handleError(error, reply) } @@ -218,6 +261,15 @@ export function registerWorktreeRoutes(app: FastifyInstance, deps: RouteDeps) { }) } +async function getManagedWorktreeIdentity(directory: string): Promise { + const [canonicalDirectory, metadata] = await Promise.all([ + realpath(directory), + lstat(path.join(directory, ".git"), { bigint: true }), + ]) + const normalizedDirectory = process.platform === "win32" ? canonicalDirectory.toLowerCase() : canonicalDirectory + return `${normalizedDirectory}\0${metadata.dev}\0${metadata.ino}\0${metadata.birthtimeNs}` +} + function handleError(error: unknown, reply: FastifyReply) { reply.code(400) return { error: error instanceof Error ? error.message : "Unable to fulfill request" } diff --git a/packages/server/src/shutdown.test.ts b/packages/server/src/shutdown.test.ts index 446e77918..befc62987 100644 --- a/packages/server/src/shutdown.test.ts +++ b/packages/server/src/shutdown.test.ts @@ -10,7 +10,7 @@ import { const logger = { info() {}, warn() {}, error() {} } const operations = (overrides: Partial = {}): ServerShutdownOperations => ({ - stopInstanceEventBridge() {}, stopSidecars() {}, stopClientConnections() {}, + stopInstanceEventBridge() {}, stopWorkflowRuns() {}, stopSidecars() {}, stopClientConnections() {}, stopRemoteProxySessions() {}, stopWorkspaces() {}, stopHttpServers() {}, stopReleaseMonitor() {}, ...overrides, }) @@ -24,7 +24,9 @@ describe("server shutdown orchestration", () => { stopWorkspaces: () => { calls.push(`workspaces-${++attempts}`); if (attempts === 1) throw new Error("still alive") }, stopHttpServers: () => { calls.push("http") }, }), logger) - assert.deepEqual(calls, ["workspaces-1", "remote-proxy", "workspaces-2", "http"]) + assert.ok(calls.indexOf("remote-proxy") < calls.indexOf("http")) + assert.ok(calls.indexOf("workspaces-1") < calls.indexOf("workspaces-2")) + assert.ok(calls.indexOf("workspaces-2") < calls.indexOf("http")) }) it("closes remaining resources and aggregates the concrete current error", async () => { @@ -45,20 +47,35 @@ describe("server shutdown orchestration", () => { assert.deepEqual([attempts, closed], [2, ["http", "release-monitor"]]) }) - it("starts workspace cleanup without waiting for preliminary shutdown", async () => { - let releasePreliminary!: () => void - const preliminary = new Promise((resolve) => { releasePreliminary = resolve }) + it("finishes workflow cancellation before workspace cleanup without blocking unrelated cleanup", async () => { + let releaseWorkflow!: () => void + const workflow = new Promise((resolve) => { releaseWorkflow = resolve }) + let releaseUnrelated!: () => void + const unrelated = new Promise((resolve) => { releaseUnrelated = resolve }) let workspaceStarted = false const shutdown = orchestrateServerShutdown(operations({ - stopRemoteProxySessions: () => preliminary, + stopWorkflowRuns: () => workflow, + stopRemoteProxySessions: () => unrelated, stopWorkspaces: () => { workspaceStarted = true }, }), logger) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(workspaceStarted, false) + releaseWorkflow() await new Promise((resolve) => setImmediate(resolve)) assert.equal(workspaceStarted, true) - releasePreliminary() + releaseUnrelated() await shutdown }) + + it("still cleans workspaces after workflow cancellation fails", async () => { + let workspaceStarted = false + await assert.rejects(orchestrateServerShutdown(operations({ + stopWorkflowRuns: () => { throw new Error("abort failed") }, + stopWorkspaces: () => { workspaceStarted = true }, + }), logger), AggregateError) + assert.equal(workspaceStarted, true) + }) }) describe("server shutdown signal boundary", () => { diff --git a/packages/server/src/shutdown.ts b/packages/server/src/shutdown.ts index 3324aa255..ab4a6a7b1 100644 --- a/packages/server/src/shutdown.ts +++ b/packages/server/src/shutdown.ts @@ -6,7 +6,7 @@ export const SERVER_SHUTDOWN_COMPLETE = "CODENOMAD_SHUTDOWN_STATUS:complete" export const SERVER_SHUTDOWN_INCOMPLETE = "CODENOMAD_SHUTDOWN_STATUS:incomplete" export type ServerShutdownOperations = Record< - "stopInstanceEventBridge" | "stopSidecars" | "stopClientConnections" | "stopRemoteProxySessions" | "stopWorkspaces" | + "stopInstanceEventBridge" | "stopWorkflowRuns" | "stopSidecars" | "stopClientConnections" | "stopRemoteProxySessions" | "stopWorkspaces" | "stopHttpServers" | "stopReleaseMonitor", ShutdownOperation > @@ -75,7 +75,7 @@ export async function orchestrateServerShutdown( } } - const workspaceShutdown = (async () => { + const workspaceShutdown = async () => { const attempts = Math.max(1, Math.floor(workspaceAttempts)) for (let attempt = 1; attempt <= attempts; attempt += 1) { const [result] = await Promise.allSettled([Promise.resolve().then(operations.stopWorkspaces)]) @@ -88,13 +88,13 @@ export async function orchestrateServerShutdown( errors.push(error) logger.error({ err: error, attempts }, "Workspace manager shutdown failed") } - })() + } await Promise.all([ settle([ ["stopInstanceEventBridge", operations.stopInstanceEventBridge], ["stopSidecars", operations.stopSidecars], ["stopClientConnections", operations.stopClientConnections], ["stopRemoteProxySessions", operations.stopRemoteProxySessions], ]), - workspaceShutdown, + settle([["stopWorkflowRuns", operations.stopWorkflowRuns]]).then(workspaceShutdown), ]) await settle([["stopHttpServers", operations.stopHttpServers], ["stopReleaseMonitor", operations.stopReleaseMonitor]]) if (errors.length) throw new AggregateError(errors, "Server shutdown failed") diff --git a/packages/server/src/workflows/definition-schema.test.ts b/packages/server/src/workflows/definition-schema.test.ts new file mode 100644 index 000000000..adbf3205d --- /dev/null +++ b/packages/server/src/workflows/definition-schema.test.ts @@ -0,0 +1,164 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" +import { + parseWorkflowDefinition, + validateWorkflowDefinition, + WORKFLOW_DEFINITION_REVISION_LIMIT, + WORKFLOW_LIMITS, +} from "./definition-schema" +import { validateJsonSchemaValue } from "./json-schema" + +describe("workflow definition schema", () => { + it("parses canonical YAML without executable tags", () => { + const parsed = parseWorkflowDefinition(` +version: 1 +id: safe +name: Safe workflow +root: + type: agent + id: work + instructions: Do the work + tools: [read] + outputSchema: + type: object + required: [result] + properties: + result: { type: string } +`) + assert.equal(parsed.definition.root.type, "agent") + assert.equal(parsed.canonical, parseWorkflowDefinition(parsed.canonical).canonical) + assert.equal(validateWorkflowDefinition("value: !!js/function function() {}" ).valid, false) + }) + + it("rejects duplicate IDs and statically excessive dynamic expansion", () => { + const duplicate = validateWorkflowDefinition({ + version: 1, id: "duplicate", name: "Duplicate", + root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "same", instructions: "One" }, + { type: "agent", id: "same", instructions: "Two" }, + ] }, + }) + assert.equal(duplicate.valid, false) + + const expansion = validateWorkflowDefinition({ + version: 1, id: "large", name: "Large", maxExpandedNodes: WORKFLOW_LIMITS.expandedNodes, + root: { + type: "foreach", id: "outer", item: "outerItem", maxItems: 101, items: [], + body: { + type: "repeat", id: "inner", maxIterations: 100, + body: { type: "agent", id: "work", instructions: "Work" }, + }, + }, + }) + assert.equal(expansion.valid, false) + + assert.equal(validateWorkflowDefinition({ + version: 1, id: "unsafe-retry", name: "Unsafe retry", + root: { type: "shell", id: "deploy", agent: "build", command: "deploy", retry: { maxAttempts: 2 } }, + }).valid, false) + }) + + it("accepts saved workflow calls with bounded input values", () => { + const result = validateWorkflowDefinition({ + version: 1, id: "caller", name: "Caller", + root: { + type: "workflow", id: "nested", definitionId: "saved", definitionRevision: 2, + inputs: { environment: "test", payload: { $ref: "inputs.payload" } }, + }, + }) + assert.equal(result.valid, true) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "bad-caller", name: "Bad caller", + root: { type: "workflow", id: "nested", definitionId: "../saved" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "future-caller", name: "Future caller", + root: { + type: "workflow", id: "nested", definitionId: "saved", + definitionRevision: WORKFLOW_DEFINITION_REVISION_LIMIT + 1, + }, + }).valid, false) + }) + + it("accepts an explicit repeat exhaustion policy", () => { + assert.equal(validateWorkflowDefinition({ + version: 1, id: "bounded-loop", name: "Bounded loop", + root: { + type: "repeat", id: "retry", maxIterations: 3, onExhausted: "fail", + body: { type: "agent", id: "work", instructions: "Try again" }, + }, + }).valid, true) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "invalid-loop", name: "Invalid loop", + root: { + type: "repeat", id: "retry", maxIterations: 3, onExhausted: "continue", + body: { type: "agent", id: "work", instructions: "Try again" }, + }, + }).valid, false) + }) + + it("accepts only portable persistent session keys", () => { + assert.equal(validateWorkflowDefinition({ + version: 1, id: "persistent-agent", name: "Persistent agent", + root: { type: "agent", id: "work", sessionKey: "luna-worker", instructions: "Continue" }, + }).valid, true) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "invalid-session", name: "Invalid session", + root: { type: "agent", id: "work", sessionKey: "not portable", instructions: "Continue" }, + }).valid, false) + }) + + it("requires portable lowercase saved definition IDs", () => { + assert.equal(validateWorkflowDefinition({ + version: 1, id: "CaseCollision", name: "Uppercase definition", + root: { type: "agent", id: "work", instructions: "Work" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "lowercase-id", name: "Lowercase definition", + root: { type: "workflow", id: "CallNode", definitionId: "UppercaseTarget" }, + }).valid, false) + assert.equal(validateWorkflowDefinition({ + version: 1, id: "lowercase-id", name: "Lowercase definition", + root: { type: "workflow", id: "CallNode", definitionId: "lowercase-target" }, + }).valid, true) + }) + + it("accepts only the JSON schema subset enforced at runtime", () => { + const definition = (outputSchema: Record) => ({ + version: 1, id: "schema", name: "Schema", + root: { type: "agent", id: "work", instructions: "Work", outputSchema }, + }) + assert.equal(validateWorkflowDefinition(definition({ + type: "object", + required: ["result"], + properties: { result: { type: "string", minLength: 1 } }, + additionalProperties: false, + })).valid, true) + assert.equal(validateWorkflowDefinition(definition({ enum: [null, false, 0, "0", [], {}] })).valid, true) + + for (const outputSchema of [ + { type: "string", pattern: "^(a+)+$" }, + { type: "string", format: "email" }, + { type: "object", properties: { value: { $ref: "#/definitions/value" } } }, + { type: "mystery" }, + { items: true }, + { additionalProperties: { type: "string" } }, + { enum: [] }, + { enum: ["same", "same"] }, + { enum: [{ left: 1, right: 2 }, { right: 2, left: 1 }] }, + ]) assert.equal(validateWorkflowDefinition(definition(outputSchema)).valid, false) + }) + + it("enforces closed objects even when no properties are declared", () => { + const closed = { type: "object", additionalProperties: false } + assert.deepEqual(validateJsonSchemaValue({}, closed), []) + assert.deepEqual(validateJsonSchemaValue({ unexpected: true }, closed), ["$.unexpected is not allowed"]) + assert.deepEqual(validateJsonSchemaValue({ value: 1 }, { + type: "object", properties: { value: { type: "integer" } }, additionalProperties: false, + }), []) + assert.deepEqual(validateJsonSchemaValue({ ordered: { right: 2, left: 1 } }, { + const: { ordered: { left: 1, right: 2 } }, + }), []) + assert.deepEqual(validateJsonSchemaValue("😀", { minLength: 1, maxLength: 1 }), []) + }) +}) diff --git a/packages/server/src/workflows/definition-schema.ts b/packages/server/src/workflows/definition-schema.ts new file mode 100644 index 000000000..f4586f18e --- /dev/null +++ b/packages/server/src/workflows/definition-schema.ts @@ -0,0 +1,279 @@ +import { parseDocument } from "yaml" +import { z } from "zod" +import type { + WorkflowCondition, + WorkflowDefinitionV1, + WorkflowNode, + WorkflowValue, +} from "../api-types" +import { inspectJsonSchema } from "./json-schema" + +export const WORKFLOW_DEFINITION_REVISION_LIMIT = 100 + +export const WORKFLOW_LIMITS = { + sourceBytes: 256_000, + staticNodes: 256, + expandedNodes: 10_000, + depth: 24, + branchWidth: 32, + concurrency: 16, + foreachItems: 1_000, + repeatIterations: 1_000, + retries: 5, + nestingDepth: 8, + timeoutMs: 24 * 60 * 60 * 1_000, + schemaBytes: 32_000, + valueDepth: 20, +} as const + +const IdSchema = z.string().trim().min(1).max(100).regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/) +const DefinitionIdSchema = z.string().trim().min(1).max(100).regex( + /^[a-z0-9][a-z0-9_-]*$/, + "Definition IDs must start with a lowercase letter or number and contain only lowercase letters, numbers, hyphens, or underscores", +) +const TitleSchema = z.string().trim().min(1).max(200) +const ModelSchema = z.object({ + providerID: z.string().trim().min(1).max(200), + modelID: z.string().trim().min(1).max(200), +}).strict() +const RefSchema = z.object({ + $ref: z.string().regex(/^(inputs|nodes|vars)(?:\.[a-zA-Z0-9_-]+)+$/).max(500), +}).strict() + +export const WorkflowValueSchema: z.ZodType = z.lazy(() => z.union([ + z.null(), + z.boolean(), + z.number().finite(), + z.string().max(50_000), + RefSchema, + z.array(WorkflowValueSchema).max(WORKFLOW_LIMITS.foreachItems), + z.record(WorkflowValueSchema), +])) + +export const WorkflowConditionSchema: z.ZodType = z.union([ + z.boolean(), + z.object({ + value: WorkflowValueSchema, + equals: WorkflowValueSchema.optional(), + notEquals: WorkflowValueSchema.optional(), + exists: z.boolean().optional(), + truthy: z.boolean().optional(), + }).strict().superRefine((condition, ctx) => { + const operators = [condition.equals, condition.notEquals, condition.exists, condition.truthy] + .filter((value) => value !== undefined) + if (operators.length > 1) ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Condition accepts one operator" }) + }), +]) + +const RetrySchema = z.object({ + maxAttempts: z.number().int().min(1).max(WORKFLOW_LIMITS.retries), + delayMs: z.number().int().min(0).max(60_000).optional(), + idempotent: z.boolean().optional(), +}).strict().superRefine((retry, ctx) => { + if (retry.maxAttempts > 1 && retry.idempotent !== true) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Retries require idempotent: true" }) + } +}) + +const JsonSchema = z.record(z.unknown()).superRefine((value, ctx) => { + try { + const serialized = JSON.stringify(value) + if (Buffer.byteLength(serialized, "utf8") > WORKFLOW_LIMITS.schemaBytes) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "JSON schema is too large" }) + } + } catch { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "JSON schema must be JSON serializable" }) + } + for (const issue of inspectJsonSchema(value)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: issue.message, path: issue.path }) + } +}) + +const NodeBase = { + id: IdSchema, + title: TitleSchema.optional(), + if: WorkflowConditionSchema.optional(), +} + +export const WorkflowNodeSchema: z.ZodType = z.lazy(() => z.union([ + z.object({ ...NodeBase, type: z.literal("sequence"), steps: z.array(WorkflowNodeSchema).min(1).max(WORKFLOW_LIMITS.branchWidth) }).strict(), + z.object({ + ...NodeBase, + type: z.literal("parallel"), + branches: z.array(WorkflowNodeSchema).min(1).max(WORKFLOW_LIMITS.branchWidth), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("foreach"), + items: WorkflowValueSchema, + item: IdSchema, + body: WorkflowNodeSchema, + maxItems: z.number().int().min(1).max(WORKFLOW_LIMITS.foreachItems), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("repeat"), + body: WorkflowNodeSchema, + maxIterations: z.number().int().min(1).max(WORKFLOW_LIMITS.repeatIterations), + while: WorkflowConditionSchema.optional(), + onExhausted: z.enum(["complete", "fail"]).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("agent"), + instructions: z.string().trim().min(1).max(50_000), + context: WorkflowValueSchema.optional(), + sessionKey: IdSchema.optional(), + agent: z.string().trim().min(1).max(200).optional(), + model: ModelSchema.optional(), + tools: z.array(z.string().trim().min(1).max(200)).max(128).refine((tools) => new Set(tools).size === tools.length, "Tool IDs must be unique").optional(), + outputSchema: JsonSchema.optional(), + retry: RetrySchema.optional(), + timeoutMs: z.number().int().min(1).max(WORKFLOW_LIMITS.timeoutMs).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("shell"), + command: z.string().trim().min(1).max(50_000), + agent: z.string().trim().min(1).max(200), + model: ModelSchema.optional(), + retry: RetrySchema.optional(), + timeoutMs: z.number().int().min(1).max(WORKFLOW_LIMITS.timeoutMs).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("gate"), + gate: z.enum(["approval", "input"]), + prompt: z.string().trim().min(1).max(20_000), + inputSchema: JsonSchema.optional(), + }).strict().superRefine((gate, ctx) => { + if (gate.gate === "approval" && gate.inputSchema) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: "Approval gates do not accept inputSchema", path: ["inputSchema"] }) + } + }), + z.object({ + ...NodeBase, + type: z.literal("workflow"), + definitionId: DefinitionIdSchema, + definitionRevision: z.number().int().min(1).max(WORKFLOW_DEFINITION_REVISION_LIMIT).optional(), + inputs: z.record(WorkflowValueSchema).optional(), + }).strict(), + z.object({ + ...NodeBase, + type: z.literal("condition"), + condition: WorkflowConditionSchema, + then: WorkflowNodeSchema, + else: WorkflowNodeSchema.optional(), + }).strict(), +]) as unknown as z.ZodType) + +const DefinitionSchemaBase = z.object({ + version: z.literal(1), + id: DefinitionIdSchema, + name: TitleSchema, + description: z.string().trim().max(2_000).optional(), + root: WorkflowNodeSchema, + budget: z.object({ + maxCost: z.number().finite().positive().max(1_000_000).optional(), + maxTokens: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + }).strict().refine((budget) => budget.maxCost !== undefined || budget.maxTokens !== undefined, "Budget cannot be empty").optional(), + maxConcurrency: z.number().int().min(1).max(WORKFLOW_LIMITS.concurrency).optional(), + maxExpandedNodes: z.number().int().min(1).max(WORKFLOW_LIMITS.expandedNodes).optional(), +}).strict() + +const inspectTree = (node: WorkflowNode, state: { ids: Set; count: number; estimated: number }, depth: number): string[] => { + const issues: string[] = [] + state.count += 1 + if (depth > WORKFLOW_LIMITS.depth) issues.push(`Node ${node.id} exceeds maximum depth ${WORKFLOW_LIMITS.depth}`) + if (state.ids.has(node.id)) issues.push(`Node ID ${node.id} is duplicated`) + state.ids.add(node.id) + + let children: WorkflowNode[] = [] + let multiplier = 1 + if (node.type === "sequence") children = node.steps + else if (node.type === "parallel") children = node.branches + else if (node.type === "foreach") { children = [node.body]; multiplier = node.maxItems } + else if (node.type === "repeat") { children = [node.body]; multiplier = node.maxIterations } + else if (node.type === "condition") children = [node.then, ...(node.else ? [node.else] : [])] + + const before = state.estimated + state.estimated += 1 + for (const child of children) issues.push(...inspectTree(child, state, depth + 1)) + if (multiplier > 1) state.estimated += (state.estimated - before - 1) * (multiplier - 1) + return issues +} + +export const WorkflowDefinitionSchema: z.ZodType = DefinitionSchemaBase.superRefine((definition, ctx) => { + const state = { ids: new Set(), count: 0, estimated: 0 } + for (const message of inspectTree(definition.root, state, 1)) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message, path: ["root"] }) + } + if (state.count > WORKFLOW_LIMITS.staticNodes) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: `Definition exceeds ${WORKFLOW_LIMITS.staticNodes} static nodes`, path: ["root"] }) + } + const expandedLimit = definition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (state.estimated > expandedLimit) { + ctx.addIssue({ code: z.ZodIssueCode.custom, message: `Definition can expand to ${state.estimated} nodes, above limit ${expandedLimit}`, path: ["root"] }) + } +}) + +const sortJson = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sortJson) + if (!value || typeof value !== "object") return value + return Object.fromEntries(Object.keys(value as Record).sort().map((key) => [ + key, + sortJson((value as Record)[key]), + ])) +} + +export interface ParsedWorkflowDefinition { + definition: WorkflowDefinitionV1 + canonical: string +} + +const inspectInput = (input: unknown) => { + const pending = [{ value: input, depth: 0 }] + const seen = new WeakSet() + let values = 0 + while (pending.length) { + const { value, depth } = pending.pop()! + if (++values > 50_000) throw new Error("Workflow definition has too many values") + if (depth > WORKFLOW_LIMITS.depth * 3) throw new Error("Workflow definition is too deeply nested") + if (value === null || typeof value === "string" || typeof value === "boolean") continue + if (typeof value === "number" && Number.isFinite(value)) continue + if (!value || typeof value !== "object") throw new Error("Workflow definition must contain only JSON values") + if (seen.has(value)) throw new Error("Workflow definition cannot contain cycles") + seen.add(value) + if (!Array.isArray(value) && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) { + throw new Error("Workflow definition must contain only plain JSON objects") + } + for (const child of Object.values(value)) pending.push({ value: child, depth: depth + 1 }) + } +} + +export function parseWorkflowDefinition(source: string | unknown): ParsedWorkflowDefinition { + let input = source + if (typeof source === "string") { + if (Buffer.byteLength(source, "utf8") > WORKFLOW_LIMITS.sourceBytes) throw new Error("Workflow definition source is too large") + const document = parseDocument(source, { schema: "core", uniqueKeys: true }) + if (document.errors.length) throw new Error(document.errors.map((error) => error.message).join("; ")) + input = document.toJS({ maxAliasCount: 0 }) + } + inspectInput(input) + if (Buffer.byteLength(JSON.stringify(input), "utf8") > WORKFLOW_LIMITS.sourceBytes) { + throw new Error("Workflow definition source is too large") + } + const definition = WorkflowDefinitionSchema.parse(input) + return { definition, canonical: `${JSON.stringify(sortJson(definition), null, 2)}\n` } +} + +export function validateWorkflowDefinition(source: string | unknown) { + try { + return { valid: true as const, ...parseWorkflowDefinition(source) } + } catch (error) { + if (error instanceof z.ZodError) return { valid: false as const, issues: error.issues } + return { valid: false as const, issues: [{ code: "custom", path: [], message: error instanceof Error ? error.message : String(error) }] } + } +} diff --git a/packages/server/src/workflows/definition-store.test.ts b/packages/server/src/workflows/definition-store.test.ts new file mode 100644 index 000000000..0bede7b8f --- /dev/null +++ b/packages/server/src/workflows/definition-store.test.ts @@ -0,0 +1,255 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it, type TestContext } from "node:test" +import { + WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT, + WORKFLOW_DEFINITION_FILE_BYTES_LIMIT, + WORKFLOW_DEFINITION_CATALOG_BYTES_LIMIT, + WORKFLOW_DEFINITION_RECORD_LIMIT, + WORKFLOW_DEFINITION_REVISION_LIMIT, + WorkflowDefinitionStore, +} from "./definition-store" + +const definition = (name: string) => ({ + version: 1 as const, + id: "stored", + name, + root: { type: "agent" as const, id: "work", instructions: "Work" }, +}) + +describe("WorkflowDefinitionStore", () => { + it("rejects definition IDs that can collide by case", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-case-")) + try { + const store = new WorkflowDefinitionStore(directory) + await assert.rejects(store.create({ ...definition("Uppercase"), id: "Stored" }), /lowercase/) + assert.equal((await store.create(definition("Lowercase"))).id, "stored") + await assert.rejects(store.get("Stored"), /Invalid workflow definition ID/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps immutable revisions and atomically persists a tombstone", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-")) + try { + const store = new WorkflowDefinitionStore(directory) + const first = await store.create(definition("First")) + const second = await store.update("stored", 1, definition("Second")) + assert.equal(first.revision, 1) + assert.equal(second.revision, 2) + assert.equal((await store.get("stored", 1))?.definition.name, "First") + assert.equal((await store.get("stored"))?.definition.name, "Second") + await assert.rejects(store.update("stored", 1, definition("Stale")), /revision is 2/) + assert.equal(await store.delete("stored", 2), true) + assert.equal(await store.get("stored"), undefined) + assert.equal(await store.get("stored", 1), undefined) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "First") + assert.equal((await store.inspectRevision("stored", 2))?.definition.name, "Second") + assert.deepEqual((await fs.readdir(directory)).filter((entry) => entry.endsWith(".tmp")), []) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("acknowledges an expected revision once across store instances", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-concurrency-")) + try { + const first = new WorkflowDefinitionStore(directory) + const second = new WorkflowDefinitionStore(directory) + await fs.mkdir(path.join(directory, ".write.lock")) + await fs.writeFile(path.join(directory, ".write.lock", "owner.json"), JSON.stringify({ + token: "stale", pid: 2_147_483_647, + })) + await first.create(definition("First")) + const updates = await Promise.allSettled([ + first.update("stored", 1, definition("First update")), + second.update("stored", 1, definition("Second update")), + ]) + assert.equal(updates.filter((result) => result.status === "fulfilled").length, 1) + assert.match((updates.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /revision is 2/) + assert.equal((await second.update("stored", 2, definition("After conflict"))).revision, 3) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds revision count without pruning immutable or tombstoned history", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-count-")) + try { + const store = new WorkflowDefinitionStore(directory) + await store.create(definition("Revision 1")) + for (let revision = 2; revision <= WORKFLOW_DEFINITION_REVISION_LIMIT; revision++) { + await store.update("stored", revision - 1, definition(`Revision ${revision}`)) + } + await assert.rejects( + store.update("stored", WORKFLOW_DEFINITION_REVISION_LIMIT, definition("Over limit")), + /revision limit reached/, + ) + assert.equal((await store.get("stored"))?.revision, WORKFLOW_DEFINITION_REVISION_LIMIT) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "Revision 1") + assert.equal(await store.delete("stored", WORKFLOW_DEFINITION_REVISION_LIMIT), true) + assert.equal(await store.get("stored"), undefined) + assert.equal((await store.inspectRevision("stored", WORKFLOW_DEFINITION_REVISION_LIMIT))?.definition.name, `Revision ${WORKFLOW_DEFINITION_REVISION_LIMIT}`) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds aggregate revision bytes without blocking tombstones", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-bytes-")) + try { + const store = new WorkflowDefinitionStore(directory) + const largeDefinition = (name: string) => ({ + version: 1 as const, + id: "stored", + name, + root: { + type: "sequence" as const, + id: "root", + steps: Array.from({ length: 5 }, (_, index) => ({ + type: "agent" as const, + id: `work-${index}`, + instructions: `${index}${"x".repeat(49_000)}`, + })), + }, + }) + let revision = (await store.create(largeDefinition("Large 1"))).revision + await assert.rejects(async () => { + while (revision < WORKFLOW_DEFINITION_REVISION_LIMIT) { + revision = (await store.update("stored", revision, largeDefinition(`Large ${revision + 1}`))).revision + } + }, /history size limit reached/) + assert.ok(revision < WORKFLOW_DEFINITION_REVISION_LIMIT) + const storedPath = path.join(directory, "stored.json") + const persisted = JSON.parse(await fs.readFile(storedPath, "utf8")) as { revisions: unknown[] } + assert.ok(persisted.revisions.reduce((total, record) => total + Buffer.byteLength(JSON.stringify(record), "utf8"), 0) <= WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) + assert.equal((await store.get("stored"))?.revision, revision) + assert.equal((await store.inspectRevision("stored", 1))?.definition.name, "Large 1") + assert.equal(await store.delete("stored", revision), true) + assert.equal((await store.inspectRevision("stored", revision))?.revision, revision) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds active and tombstoned definition records", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-records-")) + try { + await Promise.all(Array.from({ length: WORKFLOW_DEFINITION_RECORD_LIMIT }, (_, index) => + fs.writeFile(path.join(directory, `old-${index}.json`), "{}"))) + const store = new WorkflowDefinitionStore(directory) + await assert.rejects(store.create(definition("Over limit")), /record limit reached/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("reads catalog histories with bounded concurrency", async (context: TestContext) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-list-")) + try { + const store = new WorkflowDefinitionStore(directory) + for (let index = 0; index < 4; index++) { + await store.create({ ...definition(`Definition ${index}`), id: `stored-${index}` }) + } + + const readFile = fs.readFile.bind(fs) + let activeReads = 0 + let maxActiveReads = 0 + let reads = 0 + context.mock.method(fs, "readFile", async (...args: Parameters) => { + reads++ + activeReads++ + maxActiveReads = Math.max(maxActiveReads, activeReads) + await new Promise((resolve) => setTimeout(resolve, 5)) + try { + return await readFile(...args) + } finally { + activeReads-- + } + }) + + const [first, second] = await Promise.all([store.list(), store.list()]) + assert.equal(first.length, 4) + assert.equal(second.length, 4) + assert.equal(maxActiveReads, 1) + assert.equal(reads, 4) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("does not reuse an in-flight pre-write catalog after the write completes", async (context: TestContext) => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-list-write-")) + try { + const store = new WorkflowDefinitionStore(directory) + await store.create(definition("Before")) + const readFile = fs.readFile.bind(fs) + let release!: () => void + let started!: () => void + const blocked = new Promise((resolve) => { release = resolve }) + const reading = new Promise((resolve) => { started = resolve }) + let delayFirstRead = true + context.mock.method(fs, "readFile", async (...args: Parameters) => { + const contents = await readFile(...args) + if (delayFirstRead && String(args[0]).endsWith("stored.json")) { + delayFirstRead = false + started() + await blocked + } + return contents + }) + + const stale = store.list() + await reading + await store.update("stored", 1, definition("After")) + const fresh = store.list() + release() + assert.equal((await stale)[0]?.definition.name, "Before") + assert.equal((await fresh)[0]?.definition.name, "After") + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("bounds stored-file I/O and historical bytes before catalog validation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-list-bounds-")) + try { + const oversizedPath = path.join(directory, "oversized.json") + await fs.writeFile(oversizedPath, "") + await fs.truncate(oversizedPath, WORKFLOW_DEFINITION_FILE_BYTES_LIMIT + 1) + const oversized = new WorkflowDefinitionStore(directory) + await assert.rejects(oversized.list(), /stored file size limit reached/) + await fs.rm(oversizedPath) + + const store = new WorkflowDefinitionStore(directory) + await store.create(definition("Stored")) + const storedPath = path.join(directory, "stored.json") + const stored = JSON.parse(await fs.readFile(storedPath, "utf8")) + stored.revisions[0].padding = "x".repeat(WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) + await fs.writeFile(storedPath, JSON.stringify(stored)) + await assert.rejects(store.list(), /history size limit reached/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("rejects a catalog response before aggregate records exhaust memory", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-definition-store-catalog-bytes-")) + try { + const store = new WorkflowDefinitionStore(directory) + const instructions = "x".repeat(49_000) + const count = Math.ceil(WORKFLOW_DEFINITION_CATALOG_BYTES_LIMIT / 95_000) + 5 + for (let index = 0; index < count; index++) { + await store.create({ ...definition(`Large ${index}`), id: `large-${index}`, root: { + type: "agent", id: "work", instructions, + } }) + } + await assert.rejects(store.list(), /catalog size limit reached/) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workflows/definition-store.ts b/packages/server/src/workflows/definition-store.ts new file mode 100644 index 000000000..da1a41e00 --- /dev/null +++ b/packages/server/src/workflows/definition-store.ts @@ -0,0 +1,299 @@ +import { randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import path from "node:path" +import type { WorkflowDefinitionRecord } from "../api-types" +import { parseWorkflowDefinition, WORKFLOW_DEFINITION_REVISION_LIMIT } from "./definition-schema" +import { withFilesystemLock } from "./filesystem-lock" + +export { WORKFLOW_DEFINITION_REVISION_LIMIT } from "./definition-schema" + +interface StoredDefinition { + version: 1 + id: string + currentRevision: number + deletedAt?: string + revisions: WorkflowDefinitionRecord[] +} + +export class WorkflowDefinitionStoreError extends Error { + constructor(message: string, readonly statusCode: number) { + super(message) + } +} + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T +export const WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT = 4 * 1024 * 1024 +export const WORKFLOW_DEFINITION_FILE_BYTES_LIMIT = WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT * 2 +export const WORKFLOW_DEFINITION_RECORD_LIMIT = 1_000 +export const WORKFLOW_DEFINITION_CATALOG_BYTES_LIMIT = 16 * 1024 * 1024 + +const revisionBytes = (record: WorkflowDefinitionRecord) => Buffer.byteLength(JSON.stringify(record), "utf8") + +function assertHistoryCapacity(revisions: WorkflowDefinitionRecord[], next: WorkflowDefinitionRecord): void { + if (revisions.length >= WORKFLOW_DEFINITION_REVISION_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition revision limit reached", 409) + } + const bytes = revisions.reduce((total, record) => total + revisionBytes(record), revisionBytes(next)) + if (bytes > WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition history size limit reached", 409) + } +} + +export class WorkflowDefinitionStore { + private queue = Promise.resolve() + private listing?: Promise + + constructor(private readonly directory: string) {} + + async create(source: string | unknown): Promise { + return this.write(async (assertOwned) => { + const parsed = this.parse(source) + if (await this.readFile(parsed.definition.id)) { + throw new WorkflowDefinitionStoreError("Workflow definition already exists", 409) + } + await this.assertCreateCapacity() + const now = new Date().toISOString() + const record: WorkflowDefinitionRecord = { + id: parsed.definition.id, + revision: 1, + definition: parsed.definition, + canonical: parsed.canonical, + createdAt: now, + updatedAt: now, + } + assertHistoryCapacity([], record) + await this.persist({ version: 1, id: record.id, currentRevision: 1, revisions: [record] }, assertOwned) + return clone(record) + }) + } + + async update(id: string, expectedRevision: number, source: string | unknown): Promise { + return this.write(async (assertOwned) => { + const stored = await this.requireCurrent(id) + if (stored.currentRevision !== expectedRevision) { + throw new WorkflowDefinitionStoreError(`Workflow definition revision is ${stored.currentRevision}`, 409) + } + const parsed = this.parse(source) + if (parsed.definition.id !== id) throw new WorkflowDefinitionStoreError("Definition ID cannot be changed", 400) + const previous = stored.revisions.at(-1)! + const record: WorkflowDefinitionRecord = { + id, + revision: expectedRevision + 1, + definition: parsed.definition, + canonical: parsed.canonical, + createdAt: previous.createdAt, + updatedAt: new Date().toISOString(), + } + assertHistoryCapacity(stored.revisions, record) + stored.currentRevision = record.revision + stored.revisions.push(record) + await this.persist(stored, assertOwned) + return clone(record) + }) + } + + async delete(id: string, expectedRevision: number): Promise { + return this.write(async (assertOwned) => { + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) return false + if (stored.currentRevision !== expectedRevision) { + throw new WorkflowDefinitionStoreError(`Workflow definition revision is ${stored.currentRevision}`, 409) + } + stored.deletedAt = new Date().toISOString() + await this.persist(stored, assertOwned) + return true + }) + } + + async get(id: string, revision?: number): Promise { + await this.queue.catch(() => undefined) + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) return undefined + return this.selectRevision(stored, revision) + } + + async inspectRevision(id: string, revision: number): Promise { + await this.queue.catch(() => undefined) + const stored = await this.readFile(id) + if (!stored) return undefined + return this.selectRevision(stored, revision) + } + + private selectRevision(stored: StoredDefinition, revision?: number): WorkflowDefinitionRecord | undefined { + const selected = revision === undefined + ? stored.revisions.find((record) => record.revision === stored.currentRevision) + : stored.revisions.find((record) => record.revision === revision) + return selected ? clone(selected) : undefined + } + + async list(): Promise { + await this.queue.catch(() => undefined) + if (this.listing) return this.listing + const listing = this.readList() + this.listing = listing + try { + return await listing + } finally { + if (this.listing === listing) this.listing = undefined + } + } + + private async readList(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error + } + entries = entries.filter((entry) => entry.endsWith(".json")) + if (entries.length > WORKFLOW_DEFINITION_RECORD_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition record limit reached", 413) + } + const records: WorkflowDefinitionRecord[] = [] + let catalogBytes = 0 + for (const entry of entries) { + const stored = await this.readFile(entry.slice(0, -5)) + if (stored?.deletedAt) continue + const record = stored?.revisions.find((candidate) => candidate.revision === stored.currentRevision) + if (record) { + catalogBytes += revisionBytes(record) + if (catalogBytes > WORKFLOW_DEFINITION_CATALOG_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition catalog size limit reached", 413) + } + records.push(record) + } + } + return records + .sort((left, right) => left.definition.name.localeCompare(right.definition.name)) + } + + private async requireCurrent(id: string): Promise { + const stored = await this.readFile(id) + if (!stored || stored.deletedAt) throw new WorkflowDefinitionStoreError("Workflow definition not found", 404) + return stored + } + + private async assertCreateCapacity(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.directory) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (entries.filter((entry) => entry.endsWith(".json")).length >= WORKFLOW_DEFINITION_RECORD_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition record limit reached", 409) + } + } + + private async write(operation: (assertOwned: () => Promise) => Promise): Promise { + const pending = this.queue.catch(() => undefined) + .then(() => withFilesystemLock(path.join(this.directory, ".write.lock"), operation)) + .then((result) => { + this.listing = undefined + return result + }) + this.queue = pending.then(() => undefined, () => undefined) + return pending + } + + private definitionPath(id: string) { + if (!/^[a-z0-9][a-z0-9_-]{0,99}$/.test(id)) throw new WorkflowDefinitionStoreError("Invalid workflow definition ID", 400) + return path.join(this.directory, `${id}.json`) + } + + private parse(source: string | unknown) { + try { + return parseWorkflowDefinition(source) + } catch (error) { + throw new WorkflowDefinitionStoreError(error instanceof Error ? error.message : String(error), 400) + } + } + + private async readFile(id: string): Promise { + try { + const definitionPath = this.definitionPath(id) + const stat = await fs.stat(definitionPath) + if (stat.size > WORKFLOW_DEFINITION_FILE_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition stored file size limit reached", 413) + } + const source = await fs.readFile(definitionPath, "utf8") + if (Buffer.byteLength(source, "utf8") > WORKFLOW_DEFINITION_FILE_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition stored file size limit reached", 413) + } + const stored = JSON.parse(source) as StoredDefinition + if (stored.version !== 1 || stored.id !== id || !Number.isInteger(stored.currentRevision) + || stored.currentRevision < 1 || stored.currentRevision > WORKFLOW_DEFINITION_REVISION_LIMIT + || !Array.isArray(stored.revisions)) { + throw new Error(`Invalid stored workflow definition ${id}`) + } + if (stored.currentRevision !== stored.revisions.length || stored.revisions.length === 0) { + throw new Error(`Invalid stored workflow definition ${id}`) + } + let historyBytes = 0 + for (const record of stored.revisions) { + historyBytes += revisionBytes(record) + if (historyBytes > WORKFLOW_DEFINITION_HISTORY_BYTES_LIMIT) { + throw new WorkflowDefinitionStoreError("Workflow definition history size limit reached", 413) + } + } + for (const [index, record] of stored.revisions.entries()) { + if (record.id !== id || record.definition.id !== id || record.revision !== index + 1) throw new Error(`Invalid stored workflow definition ${id}`) + const parsed = parseWorkflowDefinition(record.definition) + if (record.canonical !== parsed.canonical) throw new Error(`Invalid canonical workflow definition ${id}`) + } + return stored + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } + } + + private async persist(stored: StoredDefinition, assertOwned: () => Promise): Promise { + await ensureDurableDirectory(this.directory) + const destination = this.definitionPath(stored.id) + const temporary = `${destination}.${randomUUID()}.tmp` + try { + const handle = await fs.open(temporary, "wx") + try { + await handle.writeFile(`${JSON.stringify(stored, null, 2)}\n`, "utf8") + await handle.sync() + } finally { + await handle.close() + } + await assertOwned() + await fs.rename(temporary, destination) + await syncDirectory(this.directory) + } catch (error) { + await fs.rm(temporary, { force: true }).catch(() => undefined) + throw error + } + } +} + +async function syncDirectory(directory: string): Promise { + let handle: fs.FileHandle | undefined + try { + handle = await fs.open(directory, "r") + await handle.sync() + } catch (error) { + if (!["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EBADF", "ENOSYS"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error + } finally { + await handle?.close() + } +} + +async function ensureDurableDirectory(directory: string): Promise { + const created = await fs.mkdir(directory, { recursive: true }) + if (!created) return + const firstCreated = path.resolve(created) + let current = path.resolve(directory) + while (true) { + await syncDirectory(path.dirname(current)) + if (current === firstCreated) return + const parent = path.dirname(current) + if (parent === current) return + current = parent + } +} diff --git a/packages/server/src/workflows/filesystem-lock.test.ts b/packages/server/src/workflows/filesystem-lock.test.ts new file mode 100644 index 000000000..194675839 --- /dev/null +++ b/packages/server/src/workflows/filesystem-lock.test.ts @@ -0,0 +1,145 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { it } from "node:test" +import { withFilesystemLock } from "./filesystem-lock" + +it("never overlaps replacement locks while concurrent callers reclaim a stale owner", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-stale-lock-race-")) + const lockPath = path.join(directory, "shared.lock") + await fs.mkdir(lockPath) + await fs.writeFile(path.join(lockPath, "owner.json"), JSON.stringify({ + token: "stale-owner", + pid: 2_147_483_647, + }), "utf8") + + let active = 0 + let maximum = 0 + let releaseFirst!: () => void + let firstEntered!: () => void + const firstReady = new Promise((resolve) => { firstEntered = resolve }) + const firstBlocked = new Promise((resolve) => { releaseFirst = resolve }) + const operation = async () => { + active++ + maximum = Math.max(maximum, active) + await new Promise((resolve) => setTimeout(resolve, 5)) + active-- + } + try { + const first = withFilesystemLock(lockPath, async () => { + active++ + maximum = Math.max(maximum, active) + firstEntered() + await firstBlocked + active-- + }, { waitMs: 10, staleMs: 20, pollMs: 1 }) + await firstReady + const contenders = Array.from({ length: 7 }, () => withFilesystemLock(lockPath, operation, { + waitMs: 5_000, staleMs: 20, pollMs: 1, + })) + releaseFirst() + await Promise.all([first, ...contenders]) + assert.equal(maximum, 1) + } finally { + releaseFirst?.() + await fs.rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 50 }) + } +}) + +it("fences a replaced writer and never releases its successor", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-lock-fence-")) + const lockPath = path.join(directory, "shared.lock") + let continueWriter!: () => void + let writerReady!: () => void + const ready = new Promise((resolve) => { writerReady = resolve }) + const blocked = new Promise((resolve) => { continueWriter = resolve }) + let committed = false + try { + const writer = withFilesystemLock(lockPath, async (assertOwned) => { + writerReady() + await blocked + await assertOwned() + committed = true + }) + await ready + await fs.rename(lockPath, `${lockPath}.expired`) + await fs.mkdir(lockPath) + await fs.writeFile(path.join(lockPath, "owner.json"), JSON.stringify({ token: "successor", pid: process.pid }), "utf8") + await fs.writeFile(path.join(lockPath, "heartbeat"), "successor", "utf8") + continueWriter() + + await assert.rejects(writer, /ownership was lost/) + assert.equal(committed, false) + assert.equal(JSON.parse(await fs.readFile(path.join(lockPath, "owner.json"), "utf8")).token, "successor") + } finally { + continueWriter?.() + await fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 20 }) + } +}) + +it("recovers an abandoned reclaim claim", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-lock-claim-")) + const lockPath = path.join(directory, "shared.lock") + const observed = { token: "stale-owner", serialized: JSON.stringify({ token: "stale-owner", pid: 2_147_483_647 }) } + try { + await fs.mkdir(lockPath) + await fs.writeFile(path.join(lockPath, "owner.json"), observed.serialized, "utf8") + await fs.writeFile(path.join(lockPath, "heartbeat"), "stale", "utf8") + await fs.writeFile(path.join(lockPath, ".reclaim"), JSON.stringify({ + token: "abandoned", owner: observed, createdAt: Date.now() - 1_000, + }), "utf8") + + let acquired = false + await withFilesystemLock(lockPath, async () => { acquired = true }, { waitMs: 20, staleMs: 20, pollMs: 1 }) + assert.equal(acquired, true) + } finally { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 20 }) + } +}) + +it("publishes complete owner metadata and never reclaims a suspended same-machine owner", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-live-lock-owner-")) + const lockPath = path.join(directory, "shared.lock") + try { + await fs.mkdir(lockPath) + await fs.writeFile(path.join(lockPath, "owner.json"), JSON.stringify({ + token: "live-owner", pid: process.pid, hostname: os.hostname(), + }), "utf8") + await fs.writeFile(path.join(lockPath, "heartbeat"), "suspended", "utf8") + const old = new Date(Date.now() - 60_000) + await fs.utimes(path.join(lockPath, "heartbeat"), old, old) + + await assert.rejects(withFilesystemLock(lockPath, async () => undefined, { + waitMs: 10, staleMs: 10, pollMs: 1, + }), /Timed out waiting/) + assert.equal(JSON.parse(await fs.readFile(path.join(lockPath, "owner.json"), "utf8")).token, "live-owner") + + await fs.rm(lockPath, { recursive: true }) + await withFilesystemLock(lockPath, async () => { + const owner = JSON.parse(await fs.readFile(path.join(lockPath, "owner.json"), "utf8")) + assert.equal(typeof owner.token, "string") + assert.equal(owner.pid, process.pid) + assert.equal(typeof await fs.readFile(path.join(lockPath, "heartbeat"), "utf8"), "string") + }) + } finally { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 20 }) + } +}) + +it("fails closed for an unverifiable foreign-host lock owner", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-foreign-lock-owner-")) + const lockPath = path.join(directory, "shared.lock") + try { + await fs.mkdir(lockPath) + await fs.writeFile(path.join(lockPath, "owner.json"), JSON.stringify({ + token: "foreign-owner", pid: 2_147_483_647, hostname: "another-host", + }), "utf8") + await fs.writeFile(path.join(lockPath, "heartbeat"), "stale", "utf8") + await assert.rejects(withFilesystemLock(lockPath, async () => undefined, { + waitMs: 5, staleMs: 5, pollMs: 1, + }), /Timed out waiting/) + } finally { + await fs.rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 20 }) + } +}) diff --git a/packages/server/src/workflows/filesystem-lock.ts b/packages/server/src/workflows/filesystem-lock.ts new file mode 100644 index 000000000..9576a6f7a --- /dev/null +++ b/packages/server/src/workflows/filesystem-lock.ts @@ -0,0 +1,331 @@ +import { createHash, randomUUID } from "node:crypto" +import { spawnSync } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { probePosixProcesses, probeWindowsProcesses } from "../workspaces/process-identity" + +interface LockOwner { + token: string + pid: number + hostname?: string + runtimeToken?: string + processStart?: string + bootId?: string +} + +interface ObservedOwner { + token?: string + serialized?: string +} + +interface ReclaimClaim { + token: string + owner: ObservedOwner + createdAt: number +} + +export interface FilesystemLockOptions { + waitMs?: number + staleMs?: number + pollMs?: number +} + +const WAIT_MS = 5_000 +const STALE_MS = 30_000 +const POLL_MS = 20 +const RUNTIME_TOKEN = randomUUID() +const PROCESS_IDENTITY = processIdentity(process.pid) + +export async function withFilesystemLock( + lockPath: string, + operation: (assertOwned: () => Promise) => Promise, + options: FilesystemLockOptions = {}, +): Promise { + const waitMs = options.waitMs ?? WAIT_MS + const staleMs = options.staleMs ?? STALE_MS + const pollMs = options.pollMs ?? POLL_MS + await fs.mkdir(path.dirname(lockPath), { recursive: true }) + let deadline = Date.now() + waitMs + let staleRetryGranted = false + const identity = PROCESS_IDENTITY + const owner: LockOwner = { + token: randomUUID(), pid: process.pid, hostname: os.hostname(), runtimeToken: RUNTIME_TOKEN, + ...(identity ? { processStart: identity.startTime, ...(identity.bootId ? { bootId: identity.bootId } : {}) } : {}), + } + + while (true) { + try { + if (!await publishLock(lockPath, owner)) throw Object.assign(new Error("Lock exists"), { code: "EEXIST" }) + break + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error + if (Date.now() < deadline) { + await delay(pollMs) + continue + } + if (!staleRetryGranted && await removeStaleLock(lockPath, staleMs, pollMs)) { + staleRetryGranted = true + deadline = Date.now() + waitMs + await delay(pollMs) + continue + } + if (staleRetryGranted && Date.now() < deadline) { + await delay(pollMs) + continue + } + throw new Error(`Timed out waiting for filesystem lock ${lockPath}`) + } + } + + const assertOwned = async () => { + const current = await readObservedOwner(lockPath) + if (current.token !== owner.token || await readClaim(lockPath)) { + throw new Error(`Filesystem lock ownership was lost for ${lockPath}`) + } + } + const heartbeat = setInterval(() => void heartbeatOwned(lockPath, owner.token).catch(() => undefined), staleMs / 3) + heartbeat.unref() + try { + return await operation(assertOwned) + } finally { + clearInterval(heartbeat) + await releaseOwnedLock(lockPath, owner.token, waitMs, staleMs, pollMs) + } +} + +async function publishLock(lockPath: string, owner: LockOwner): Promise { + const prepared = `${lockPath}.prepared.${owner.token}` + try { + await fs.mkdir(prepared) + await fs.writeFile(path.join(prepared, "owner.json"), JSON.stringify(owner), { encoding: "utf8", flag: "wx" }) + await fs.writeFile(path.join(prepared, "heartbeat"), randomUUID(), { encoding: "utf8", flag: "wx" }) + await fs.rename(prepared, lockPath) + return true + } catch (error) { + if (["EEXIST", "ENOTEMPTY", "EPERM"].includes((error as NodeJS.ErrnoException).code ?? "")) return false + throw error + } finally { + await fs.rm(prepared, { recursive: true, force: true }).catch(() => undefined) + } +} + +async function removeStaleLock(lockPath: string, staleMs: number, pollMs: number): Promise { + let stale = false + let observed: ObservedOwner = {} + try { + const serialized = await fs.readFile(path.join(lockPath, "owner.json"), "utf8") + const owner = JSON.parse(serialized) as LockOwner + observed = { token: owner.token, serialized } + stale = !await ownerIsAlive(owner, lockPath, staleMs, pollMs) + } catch { + const first = await heartbeatState(lockPath) + await delay(staleMs) + stale = await heartbeatState(lockPath) === first + } + if (!stale) { + const claim = await readClaim(lockPath) + if (claim && Date.now() - claim.createdAt < staleMs) return true + if (claim) await removeOwnedClaim(lockPath, claim.token) + return !sameObservedOwner(await readObservedOwner(lockPath), observed) + } + + const claim = await acquireClaim(lockPath, observed, staleMs) + if (!claim) return true + const quarantinePath = `${lockPath}.stale.${claim.token}` + let moved = false + try { + const current = await readObservedOwner(lockPath) + const currentClaim = await readClaim(lockPath) + if (!sameObservedOwner(current, observed) || currentClaim?.token !== claim.token) return true + await fs.rename(lockPath, quarantinePath) + moved = true + const movedOwner = await readObservedOwner(quarantinePath) + const movedClaim = await readClaim(quarantinePath) + if (!sameObservedOwner(movedOwner, observed) || movedClaim?.token !== claim.token) { + await fs.rename(quarantinePath, lockPath).then(() => { moved = false }).catch(() => undefined) + return true + } + await fs.rm(quarantinePath, { recursive: true, force: true }) + return true + } catch (error) { + if (["ENOENT", "EPERM", "EACCES", "EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) return true + throw error + } finally { + if (!moved) await removeOwnedClaim(lockPath, claim.token) + } +} + +async function acquireClaim(lockPath: string, owner: ObservedOwner, staleMs: number): Promise { + const claimPath = path.join(lockPath, ".reclaim") + for (let attempt = 0; attempt < 3; attempt += 1) { + const claim: ReclaimClaim = { token: randomUUID(), owner, createdAt: Date.now() } + try { + await fs.writeFile(claimPath, JSON.stringify(claim), { encoding: "utf8", flag: "wx" }) + return claim + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error + } + const existing = await readClaim(lockPath) + if (!existing || Date.now() - existing.createdAt < staleMs) return undefined + const abandoned = `${claimPath}.abandoned.${randomUUID()}` + try { + await fs.rename(claimPath, abandoned) + const moved = await readClaimFile(abandoned) + if (moved?.token !== existing.token) return undefined + } catch (error) { + if (!["ENOENT", "EPERM", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error + } finally { + await fs.rm(abandoned, { force: true }).catch(() => undefined) + } + } + return undefined +} + +async function releaseOwnedLock(lockPath: string, token: string, waitMs: number, staleMs: number, pollMs: number): Promise { + const deadline = Date.now() + waitMs + while (true) { + const observed = await readObservedOwner(lockPath) + if (observed.token !== token) return + const claim = await acquireClaim(lockPath, observed, staleMs) + if (!claim) { + if (Date.now() >= deadline) return + await delay(pollMs) + continue + } + const releasedPath = `${lockPath}.released.${claim.token}` + let moved = false + try { + if ((await readObservedOwner(lockPath)).token !== token || (await readClaim(lockPath))?.token !== claim.token) return + await fs.rename(lockPath, releasedPath) + moved = true + if ((await readObservedOwner(releasedPath)).token !== token) { + await fs.rename(releasedPath, lockPath).then(() => { moved = false }).catch(() => undefined) + return + } + await fs.rm(releasedPath, { recursive: true, force: true }) + return + } catch (error) { + if (["ENOENT", "EPERM", "EACCES", "EEXIST", "ENOTEMPTY"].includes((error as NodeJS.ErrnoException).code ?? "")) return + throw error + } finally { + if (!moved) await removeOwnedClaim(lockPath, claim.token) + } + } +} + +async function heartbeatOwned(lockPath: string, token: string): Promise { + if ((await readObservedOwner(lockPath)).token !== token || await readClaim(lockPath)) return + await fs.writeFile(path.join(lockPath, "heartbeat"), randomUUID(), "utf8") +} + +async function readClaim(lockPath: string): Promise { + return readClaimFile(path.join(lockPath, ".reclaim")) +} + +async function readClaimFile(claimPath: string): Promise { + try { + const serialized = await fs.readFile(claimPath, "utf8") + try { + const claim = JSON.parse(serialized) as ReclaimClaim + if (typeof claim.token === "string" && Number.isFinite(claim.createdAt)) return claim + } catch { + // Malformed claims are treated as an expiring generation, never removed blindly. + } + const stat = await fs.stat(claimPath) + return { token: `malformed-${createHash("sha256").update(serialized).digest("hex")}`, owner: { serialized }, createdAt: stat.mtimeMs } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } +} + +async function removeOwnedClaim(lockPath: string, token: string): Promise { + const claim = await readClaim(lockPath) + if (claim?.token === token) await fs.rm(path.join(lockPath, ".reclaim"), { force: true }).catch(() => undefined) +} + +async function readObservedOwner(lockPath: string): Promise { + try { + const serialized = await fs.readFile(path.join(lockPath, "owner.json"), "utf8") + try { + return { token: (JSON.parse(serialized) as LockOwner).token, serialized } + } catch { + return { serialized } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return {} + throw error + } +} + +function sameObservedOwner(left: ObservedOwner, right: ObservedOwner): boolean { + return left.token !== undefined || right.token !== undefined + ? left.token !== undefined && left.token === right.token + : left.serialized === right.serialized +} + +async function ownerIsAlive(owner: LockOwner, lockPath: string, staleMs: number, pollMs: number): Promise { + if (owner.runtimeToken === RUNTIME_TOKEN) return true + if (owner.hostname && owner.hostname !== os.hostname()) return true + const liveness = processLiveness(owner) + if (liveness !== "unknown") return liveness === "alive" + const first = await heartbeatState(lockPath) + const deadline = Date.now() + staleMs + while (Date.now() < deadline) { + await delay(Math.min(Math.max(pollMs, 1), deadline - Date.now())) + try { + const current = JSON.parse(await fs.readFile(path.join(lockPath, "owner.json"), "utf8")) as LockOwner + if (current.token !== owner.token) return true + } catch { + return true + } + if (await heartbeatState(lockPath) !== first) return true + } + return false +} + +async function heartbeatState(lockPath: string): Promise { + try { + return await fs.readFile(path.join(lockPath, "heartbeat"), "utf8") + } catch { + try { + return String((await fs.stat(lockPath)).mtimeMs) + } catch { + return "missing" + } + } +} + +function processIdentity(pid: number): { startTime: string; bootId?: string } | undefined { + const snapshot = process.platform === "win32" + ? probeWindowsProcesses(spawnSync, 1_000) + : probePosixProcesses(spawnSync, 1_000, process.platform, { pids: [pid] }) + const identity = snapshot.ok ? snapshot.processes.get(pid) : undefined + return identity && { startTime: identity.startTime, ...(identity.bootId ? { bootId: identity.bootId } : {}) } +} + +function processLiveness(owner: LockOwner): "alive" | "dead" | "unknown" { + try { + process.kill(owner.pid, 0) + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH" ? "dead" : "unknown" + } + if (!owner.processStart) return "alive" + if (owner.pid === process.pid && PROCESS_IDENTITY) { + return PROCESS_IDENTITY.startTime === owner.processStart && (!owner.bootId || PROCESS_IDENTITY.bootId === owner.bootId) + ? "alive" : "dead" + } + const snapshot = process.platform === "win32" + ? probeWindowsProcesses(spawnSync, 1_000) + : probePosixProcesses(spawnSync, 1_000, process.platform, { pids: [owner.pid] }) + if (snapshot.ok) { + const current = snapshot.processes.get(owner.pid) + if (!current) return "dead" + return current.startTime === owner.processStart && (!owner.bootId || current.bootId === owner.bootId) ? "alive" : "dead" + } + return "unknown" +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/packages/server/src/workflows/interpreter.ts b/packages/server/src/workflows/interpreter.ts new file mode 100644 index 000000000..466a83903 --- /dev/null +++ b/packages/server/src/workflows/interpreter.ts @@ -0,0 +1,894 @@ +import { randomUUID } from "node:crypto" +import { isDeepStrictEqual } from "node:util" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { + WorkflowCondition, + WorkflowBudget, + WorkflowExecutionNode, + WorkflowNode, + WorkflowRun, + WorkflowUsage, + WorkflowValue, +} from "../api-types" +import { WORKFLOW_LIMITS } from "./definition-schema" +import { validateJsonSchemaValue } from "./json-schema" +import { isConfirmedRetryCheckpoint } from "./run-state" + +const DEFAULT_TIMEOUT_MS = 30 * 60 * 1_000 +const MAX_OUTPUT_CHARS = 16_000 +const MAX_RUN_OUTPUT_CHARS = 4_000_000 +const MAX_CONTEXT_BYTES = 256_000 +const MAX_CONTEXT_VALUES = 50_000 +export class WorkflowSuspendedError extends Error {} +export class WorkflowBudgetError extends Error {} +export class WorkflowCheckpointError extends Error {} +class WorkflowAmbiguousSideEffectError extends Error {} +class WorkflowRetryCleanupError extends Error {} + +export interface WorkflowInterpreterOptions { + run: WorkflowRun + client: OpencodeClient + persist: () => Promise + signal: (timeoutMs: number) => AbortSignal + sessionStarted: (sessionId: string) => boolean + sessionFinished: (sessionId: string) => void + sessionCreationStarted?: () => void + sessionCreationFinished?: () => void + abortSession: (sessionId: string) => Promise + isCancelled: () => boolean + revalidateFence?: () => Promise + isPauseCommitted?: () => boolean +} + +interface ExecutionContext { + vars: Record + inputs: Record + budgets: WorkflowBudget[] + limiters: ActionLimiter[] + definitionInvocationKey: string + instanceKey?: string +} + +interface ActionLimiter { + max: number + active: number + waiters: Array<{ + resolve: () => void + reject: (reason?: unknown) => void + signal: AbortSignal + abort: () => void + }> +} + +interface ActionResult { + output: unknown + sessionId: string +} + +const emptyUsage = (): WorkflowUsage => ({ + cost: 0, + tokens: 0, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}) + +export class WorkflowInterpreter { + private readonly run: WorkflowRun + private readonly nodes: WorkflowExecutionNode[] + private expanded = 0 + private outputChars = 0 + private readonly schedulerAbort = new AbortController() + private readonly maxConcurrency: number + private readonly rootLimiter: ActionLimiter + private readonly budgetLimiters = new Map() + private readonly sessionLimiters = new Map() + private readonly savedDefinitionKeys: Set + + constructor(private readonly options: WorkflowInterpreterOptions) { + this.run = options.run + this.nodes = this.run.executionNodes ??= [] + this.run.usage ??= emptyUsage() + this.expanded = this.nodes.length + this.outputChars = this.nodes.reduce((total, node) => total + (node.output === undefined ? 0 : JSON.stringify(node.output).length), 0) + this.maxConcurrency = this.run.definitionSnapshot?.maxConcurrency ?? 1 + this.rootLimiter = { max: this.maxConcurrency, active: 0, waiters: [] } + this.savedDefinitionKeys = new Set((this.run.savedDefinitionSnapshots ?? []).map((snapshot) => `${snapshot.id}@${snapshot.revision}`)) + } + + async execute(): Promise { + const definition = this.run.definitionSnapshot + if (!definition) throw new Error("Workflow definition snapshot is missing") + if (!this.run.rootSessionId) { + this.options.sessionCreationStarted?.() + let root: { id: string } + try { + root = await this.requireData(this.options.client.session.create({ + ...(this.run.initiatorSessionId ? { parentID: this.run.initiatorSessionId } : {}), + title: `Workflow: ${this.run.objective.slice(0, 80)}`, + metadata: this.sessionMetadata("workflow"), + }, { signal: this.operationSignal(DEFAULT_TIMEOUT_MS) }), "create workflow session") + this.options.sessionStarted(root.id) + } finally { + this.options.sessionCreationFinished?.() + } + try { + this.throwIfCancelled() + this.run.rootSessionId = root.id + await this.options.persist() + this.options.sessionFinished(root.id) + } catch (error) { + if (!await this.options.abortSession(root.id)) { + throw new WorkflowAmbiguousSideEffectError(`Workflow root session abort was not confirmed: ${this.errorMessage(error)}`) + } + this.options.sessionFinished(root.id) + throw error + } + } + const definitionInvocationKey = `${this.run.definitionId}@${this.run.definitionRevision}` + await this.executeNode(definition.root, definition.root.id, { + vars: {}, + inputs: this.run.inputs ?? {}, + budgets: definition.budget ? [definition.budget] : [], + // ponytail: usage is observed after actions, so budgeted actions serialize until nodes declare reservable maxima. + limiters: [this.rootLimiter, ...(definition.budget ? [this.budgetLimiter(definitionInvocationKey)] : [])], + definitionInvocationKey, + }) + } + + private async executeNode(node: WorkflowNode, instanceKey: string, context: ExecutionContext, parentInstanceKey?: string): Promise { + this.throwIfCancelled() + const existing = this.nodes.find((candidate) => candidate.instanceKey === instanceKey) + if (existing?.status === "completed" || existing?.status === "skipped") return existing.output + await this.pauseIfRequested() + + const execution = existing ?? this.addExecution(node, instanceKey, context.definitionInvocationKey, parentInstanceKey) + const scopedContext = { ...context, instanceKey } + if (node.if !== undefined && !this.evaluateCondition(node.if, scopedContext)) { + execution.status = "skipped" + execution.completedAt = new Date().toISOString() + await this.options.persist() + return undefined + } + + if (!isConfirmedRetryCheckpoint(execution)) { + execution.status = "running" + execution.startedAt ??= new Date().toISOString() + delete execution.error + await this.options.persist() + } + + let actionSignal: AbortSignal | undefined + if (node.type === "agent" || node.type === "shell") { + actionSignal = this.operationSignal(node.timeoutMs ?? DEFAULT_TIMEOUT_MS) + } + let sessionPermit: ActionLimiter | undefined + if (node.type === "agent" && node.sessionKey) { + sessionPermit = this.sessionLimiter(node.sessionKey) + await this.acquirePermit(sessionPermit, actionSignal!) + } + let actionSessionId: string | undefined + let nodeCompleted = false + try { + let output: unknown + switch (node.type) { + case "sequence": + output = await this.executeSequence(node.steps, instanceKey, scopedContext) + break + case "parallel": + output = await this.executeParallel(node.branches, instanceKey, scopedContext, node.maxConcurrency) + break + case "foreach": + output = await this.executeForeach(node, instanceKey, scopedContext) + break + case "repeat": + output = await this.executeRepeat(node, instanceKey, scopedContext) + break + case "condition": { + const selected = this.evaluateCondition(node.condition, scopedContext) ? node.then : node.else + output = selected ? await this.executeNode(selected, `${instanceKey}/${selected.id}`, scopedContext, instanceKey) : undefined + break + } + case "gate": + return await this.executeGate(node, execution) + case "agent": + ({ output, sessionId: actionSessionId } = await this.executeAgent(node, execution, scopedContext, actionSignal!)) + break + case "shell": + ({ output, sessionId: actionSessionId } = await this.executeShell(node, execution, scopedContext, actionSignal!)) + break + case "workflow": + output = await this.executeSavedWorkflow(node, instanceKey, scopedContext) + break + } + const structural = !["agent", "shell", "gate"].includes(node.type) + const bounded = this.boundOutput(output, structural) + if (bounded.truncated) throw new Error(`Workflow node ${node.id} output exceeds ${MAX_OUTPUT_CHARS} characters`) + const outputChars = output === undefined ? 0 : JSON.stringify(output).length + if (this.outputChars + outputChars > MAX_RUN_OUTPUT_CHARS) throw new Error("Workflow run output limit exceeded") + this.outputChars += outputChars + execution.output = bounded.output + execution.status = "completed" + execution.completedAt = new Date().toISOString() + nodeCompleted = true + await this.options.persist() + if (actionSessionId) this.options.sessionFinished(actionSessionId) + return output + } catch (caught) { + let error = caught + if (nodeCompleted) { + if (actionSessionId) this.options.sessionFinished(actionSessionId) + throw new WorkflowCheckpointError(`Workflow node completed, but its checkpoint could not be persisted: ${this.errorMessage(error)}`) + } else if (actionSessionId) { + if (await this.options.abortSession(actionSessionId)) { + this.options.sessionFinished(actionSessionId) + this.forgetSession(execution, actionSessionId) + await this.persistSessionCleanup(execution, error) + } + else error = new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + } + if (error instanceof WorkflowSuspendedError) { + execution.status = "waiting" + await this.options.persist() + } else if (error instanceof WorkflowAmbiguousSideEffectError) { + execution.status = "interrupted" + execution.error = this.errorMessage(error) + execution.completedAt = new Date().toISOString() + await this.options.persist() + } else if (!this.options.isCancelled()) { + execution.status = "failed" + execution.error = this.errorMessage(error) + execution.completedAt = new Date().toISOString() + await this.options.persist() + } + throw error + } finally { + if (sessionPermit) this.releasePermit(sessionPermit) + } + } + + private async executeSequence(steps: WorkflowNode[], parent: string, context: ExecutionContext): Promise { + let output: unknown + for (const child of steps) output = await this.executeNode(child, `${parent}/${child.id}`, context, parent) + return output + } + + private async executeParallel(branches: WorkflowNode[], parent: string, context: ExecutionContext, concurrency?: number): Promise { + return this.mapBounded(branches, concurrency, (branch) => + this.executeNode(branch, `${parent}/${branch.id}`, { + vars: { ...context.vars }, inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, + }, parent)) + } + + private async executeForeach(node: Extract, parent: string, context: ExecutionContext): Promise { + const value = this.resolveValue(node.items, context) + if (!Array.isArray(value)) throw new Error(`Foreach node ${node.id} items must resolve to an array`) + if (value.length > node.maxItems) throw new Error(`Foreach node ${node.id} exceeded maxItems ${node.maxItems}`) + return this.mapBounded(value, node.maxConcurrency, (item, index) => this.executeNode( + node.body, + `${parent}/${node.body.id}[${index}]`, + { + vars: { ...context.vars, [node.item]: item, [`${node.item}Index`]: index }, + inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, + }, + parent, + )) + } + + private async executeRepeat(node: Extract, parent: string, context: ExecutionContext): Promise { + const output: unknown[] = [] + let exhausted = true + for (let index = 0; index < node.maxIterations; index += 1) { + const instanceKey = `${parent}/${node.body.id}[${index}]` + const iterationContext = { + vars: { ...context.vars, iteration: index }, inputs: context.inputs, budgets: context.budgets, limiters: context.limiters, + definitionInvocationKey: context.definitionInvocationKey, instanceKey: context.instanceKey, + } + const existing = this.nodes.find((candidate) => candidate.instanceKey === instanceKey) + if (!existing && node.while !== undefined && !this.evaluateCondition(node.while, iterationContext)) { + exhausted = false + break + } + output.push(await this.executeNode(node.body, instanceKey, iterationContext, parent)) + } + if (exhausted && node.while !== undefined) { + const finalContext = { + vars: { ...context.vars, iteration: node.maxIterations }, inputs: context.inputs, budgets: context.budgets, + limiters: context.limiters, definitionInvocationKey: context.definitionInvocationKey, instanceKey: context.instanceKey, + } + exhausted = this.evaluateCondition(node.while, finalContext) + } + if (exhausted && node.onExhausted === "fail") { + throw new Error(`Repeat node ${node.id} exhausted ${node.maxIterations} iterations`) + } + return output + } + + private async executeGate(node: Extract, execution: WorkflowExecutionNode): Promise { + if (execution.status === "completed") return execution.output + if (this.run.pendingGate && this.run.pendingGate.executionNodeId !== execution.id) throw new WorkflowSuspendedError() + execution.status = "waiting" + this.run.pendingGate = { + executionNodeId: execution.id, + definitionNodeId: node.id, + gate: node.gate, + prompt: node.prompt, + ...(node.inputSchema ? { inputSchema: node.inputSchema } : {}), + } + this.run.status = node.gate === "approval" ? "waiting_for_review" : "waiting_for_input" + await this.options.persist() + throw new WorkflowSuspendedError() + } + + private async executeAgent( + node: Extract, + execution: WorkflowExecutionNode, + context: ExecutionContext, + signal: AbortSignal, + ): Promise { + const contextValue = node.context === undefined ? undefined : this.resolveContext(node.context, context, signal) + const prompt = [ + `Workflow node: ${node.title ?? node.id}`, + "", + `Objective:\n${this.run.objective}`, + "", + `Instructions:\n${node.instructions}`, + ...(contextValue === undefined ? [] : ["", `Context:\n${JSON.stringify(contextValue, null, 2)}`]), + ].join("\n") + return this.withActionPermit(context.limiters, signal, async () => { + this.enforceActionAdmission(context.budgets) + const tools = node.tools === undefined ? undefined : await this.toolOverrides(node.tools, signal) + return this.retry(node.retry?.maxAttempts ?? 1, node.retry?.delayMs ?? 0, execution, context.budgets, signal, async (attempt) => { + const sessionId = await this.createChildSession(node, execution, signal) + execution.attempt = attempt + await this.options.persist() + signal.throwIfAborted() + await this.options.revalidateFence?.() + signal.throwIfAborted() + try { + const response = await this.requireData(this.options.client.session.prompt({ + sessionID: sessionId, + ...(node.agent ? { agent: node.agent } : {}), + ...(node.model ? { model: node.model } : {}), + ...(tools ? { tools } : {}), + ...(node.outputSchema ? { format: { type: "json_schema" as const, schema: node.outputSchema, retryCount: 2 } } : {}), + parts: [{ type: "text", text: prompt }], + }, { signal }), `run ${node.id} session`) + this.observeUsage(response.info, execution, context.budgets) + if (response.info.error) throw new Error(this.errorMessage(response.info.error)) + if (node.outputSchema) { + if (response.info.structured === undefined) throw new Error(`Structured output is missing for ${node.id}`) + const issues = validateJsonSchemaValue(response.info.structured, node.outputSchema) + if (issues.length) throw new Error(`Structured output is invalid for ${node.id}: ${issues.join("; ")}`) + return { output: response.info.structured, sessionId } + } + return { output: response.parts.filter((part) => part.type === "text").map((part) => part.text).join("\n"), sessionId } + } catch (error) { + if (await this.options.abortSession(sessionId)) { + this.options.sessionFinished(sessionId) + this.forgetSession(execution, sessionId) + await this.persistSessionCleanup(execution, error) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + }) + }) + } + + private async executeShell( + node: Extract, + execution: WorkflowExecutionNode, + context: ExecutionContext, + signal: AbortSignal, + ): Promise { + return this.withActionPermit(context.limiters, signal, () => this.retry( + node.retry?.maxAttempts ?? 1, node.retry?.delayMs ?? 0, execution, context.budgets, signal, async (attempt) => { + this.enforceActionAdmission(context.budgets) + const sessionId = await this.createChildSession(node, execution, signal) + execution.attempt = attempt + await this.options.persist() + signal.throwIfAborted() + await this.options.revalidateFence?.() + signal.throwIfAborted() + try { + const response = await this.requireData(this.options.client.session.shell({ + sessionID: sessionId, + agent: node.agent, + command: node.command, + ...(node.model ? { model: node.model } : {}), + }, { signal }), `run ${node.id} shell`) + this.observeUsage(response.info, execution, context.budgets) + if ("error" in response.info && response.info.error) throw new Error(this.errorMessage(response.info.error)) + const toolParts = response.parts.filter((part) => part.type === "tool") + const toolError = toolParts.find((part) => part.state.status === "error") + if (toolError?.state.status === "error") throw new Error(toolError.state.error) + const output = toolParts.filter((part) => part.state.status === "completed") + .map((part) => part.state.status === "completed" ? part.state.output : "").join("\n") + return { output: output || response.parts.filter((part) => part.type === "text").map((part) => part.text).join("\n"), sessionId } + } catch (error) { + if (await this.options.abortSession(sessionId)) { + this.options.sessionFinished(sessionId) + this.forgetSession(execution, sessionId) + await this.persistSessionCleanup(execution, error) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + })) + } + + private async executeSavedWorkflow( + node: Extract, + instanceKey: string, + context: ExecutionContext, + ): Promise { + const snapshot = this.run.savedDefinitionSnapshots?.find((candidate) => + candidate.id === node.definitionId && candidate.revision === node.definitionRevision) + if (!snapshot) throw new Error(`Saved workflow snapshot ${node.definitionId}@${node.definitionRevision ?? "unresolved"} is missing`) + const inputs = Object.fromEntries(Object.entries(node.inputs ?? {}).map(([key, value]) => [key, this.resolveValue(value, context)])) + const root = snapshot.definition.root + const definitionInvocationKey = `${instanceKey}/${snapshot.id}@${snapshot.revision}` + return this.executeNode(root, `${definitionInvocationKey}/${root.id}`, { + vars: {}, + inputs, + budgets: snapshot.definition.budget ? [...context.budgets, snapshot.definition.budget] : context.budgets, + limiters: [ + ...context.limiters, + this.actionLimiter(snapshot.definition.maxConcurrency ?? 1), + ...(snapshot.definition.budget ? [this.budgetLimiter(`${snapshot.id}@${snapshot.revision}`)] : []), + ], + definitionInvocationKey, + }, instanceKey) + } + + private async createChildSession( + node: Extract, + execution: WorkflowExecutionNode, + signal: AbortSignal, + ): Promise { + this.throwIfCancelled() + const sessionKey = node.type === "agent" ? node.sessionKey : undefined + const bindings = this.run.sessionBindings + let sessionId = sessionKey && bindings && Object.prototype.hasOwnProperty.call(bindings, sessionKey) + ? bindings[sessionKey] + : undefined + let accepted: boolean | undefined + if (!sessionId) { + const bindingLimit = this.run.definitionSnapshot?.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (sessionKey && Object.keys(this.run.sessionBindings ?? {}).length >= bindingLimit) { + throw new Error(`Workflow exceeded session binding limit ${bindingLimit}`) + } + this.options.sessionCreationStarted?.() + try { + const session = await this.requireData(this.options.client.session.create({ + parentID: this.run.rootSessionId, + title: `${node.title ?? node.id}: ${this.run.objective.slice(0, 60)}`, + ...(node.agent ? { agent: node.agent } : {}), + metadata: this.sessionMetadata(node.id), + }, { signal }), `create ${node.id} session`) + sessionId = session.id + accepted = this.options.sessionStarted(sessionId) + } finally { + this.options.sessionCreationFinished?.() + } + if (sessionKey) (this.run.sessionBindings ??= {})[sessionKey] = sessionId + } + execution.sessionIds ??= [] + if (!execution.sessionIds.includes(sessionId)) execution.sessionIds.push(sessionId) + execution.status = "running" + accepted ??= this.options.sessionStarted(sessionId) + try { + if (!accepted) this.throwIfCancelled() + signal.throwIfAborted() + await this.options.persist() + signal.throwIfAborted() + } catch (error) { + if (await this.options.abortSession(sessionId)) { + this.options.sessionFinished(sessionId) + this.forgetSession(execution, sessionId) + await this.persistSessionCleanup(execution, error) + } + else throw new WorkflowAmbiguousSideEffectError(`Workflow session abort was not confirmed: ${this.errorMessage(error)}`) + throw error + } + return sessionId + } + + private async retry( + attempts: number, + delayMs: number, + execution: WorkflowExecutionNode, + budgets: WorkflowBudget[], + signal: AbortSignal, + operation: (attempt: number) => Promise, + ): Promise { + let lastError: unknown = new Error("Workflow retry limit was already exhausted") + for (let attempt = execution.attempt + 1; attempt <= attempts; attempt += 1) { + signal.throwIfAborted() + this.enforceActionAdmission(budgets) + try { + return await operation(attempt) + } catch (error) { + lastError = error + if (error instanceof WorkflowAmbiguousSideEffectError || error instanceof WorkflowBudgetError + || error instanceof WorkflowRetryCleanupError) throw error + this.throwIfCancelled() + signal.throwIfAborted() + this.enforceActionAdmission(budgets) + if (attempt < attempts && delayMs) await this.sleep(delayMs, signal) + } + } + throw lastError + } + + private async mapBounded(items: T[], requested: number | undefined, operation: (item: T, index: number) => Promise): Promise { + const concurrency = Math.min(requested ?? this.maxConcurrency, this.maxConcurrency, WORKFLOW_LIMITS.concurrency, items.length || 1) + const results = new Array(items.length) + let next = 0 + let suspended: WorkflowSuspendedError | undefined + let failed: unknown + const workers = Array.from({ length: concurrency }, async () => { + while (next < items.length && !suspended && failed === undefined) { + const index = next++ + try { + results[index] = await operation(items[index]!, index) + } catch (error) { + if (error instanceof WorkflowSuspendedError) suspended = error + else if (failed === undefined) { failed = error; this.schedulerAbort.abort(error) } + } + } + }) + await Promise.all(workers) + if (failed !== undefined) throw failed + if (suspended) throw suspended + return results + } + + private addExecution( + node: WorkflowNode, + instanceKey: string, + definitionInvocationKey: string, + parentInstanceKey?: string, + ): WorkflowExecutionNode { + const limit = this.run.definitionSnapshot?.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (++this.expanded > limit) throw new Error(`Workflow exceeded expanded node limit ${limit}`) + const execution: WorkflowExecutionNode = { + id: randomUUID(), + instanceKey, + definitionInvocationKey, + definitionNodeId: node.id, + type: node.type, + status: "pending", + attempt: 0, + ...(parentInstanceKey ? { parentInstanceKey } : {}), + } + this.nodes.push(execution) + return execution + } + + private resolveValue(value: WorkflowValue, context: ExecutionContext, visit?: () => void): unknown { + visit?.() + if (Array.isArray(value)) return value.map((item) => this.resolveValue(item, context, visit)) + if (!value || typeof value !== "object") return value + if (Object.prototype.hasOwnProperty.call(value, "$ref") && typeof value.$ref === "string") return this.resolveRef(value.$ref, context) + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, this.resolveValue(item, context, visit)])) + } + + private resolveContext(value: WorkflowValue, context: ExecutionContext, signal: AbortSignal): unknown { + const resolved = this.resolveValue(value, context, () => signal.throwIfAborted()) + const pending: Array<{ value: unknown; exit?: boolean }> = [{ value: resolved }] + const ancestors = new WeakSet() + let count = 0 + while (pending.length) { + signal.throwIfAborted() + const { value: current, exit } = pending.pop()! + if (exit) { + ancestors.delete(current as object) + continue + } + if (++count > MAX_CONTEXT_VALUES) throw new Error(`Workflow context exceeds ${MAX_CONTEXT_VALUES} values`) + if (!current || typeof current !== "object") continue + if (ancestors.has(current)) throw new Error("Workflow context contains a cycle") + ancestors.add(current) + pending.push({ value: current, exit: true }) + pending.push(...Object.values(current).reverse().map((value) => ({ value }))) + } + const serialized = JSON.stringify(resolved) + if (serialized !== undefined && Buffer.byteLength(serialized, "utf8") > MAX_CONTEXT_BYTES) { + throw new Error(`Workflow context exceeds ${MAX_CONTEXT_BYTES} bytes`) + } + return resolved + } + + private resolveRef(ref: string, context: ExecutionContext): unknown { + const [root, name, ...path] = ref.split(".") + let value: unknown + if (root === "inputs" && Object.prototype.hasOwnProperty.call(context.inputs, name)) value = context.inputs[name!] + else if (root === "vars" && Object.prototype.hasOwnProperty.call(context.vars, name)) value = context.vars[name!] + else if (root === "nodes") { + const candidates = this.nodes.filter((node) => node.definitionNodeId === name && node.status === "completed" + && this.executionDefinitionInvocationKey(node) === context.definitionInvocationKey + && this.referenceScopeMatches(node.instanceKey, context.instanceKey)).reverse() + value = candidates.sort((left, right) => this.commonPrefix(right.instanceKey, context.instanceKey ?? "") + - this.commonPrefix(left.instanceKey, context.instanceKey ?? "")).at(0) + } + for (const part of path) { + if (!value || typeof value !== "object" || !Object.prototype.hasOwnProperty.call(value, part)) return undefined + value = (value as Record)[part] + } + return value + } + + private evaluateCondition(condition: WorkflowCondition, context: ExecutionContext): boolean { + if (typeof condition === "boolean") return condition + const actual = this.resolveValue(condition.value, context) + if (condition.equals !== undefined) return this.equal(actual, this.resolveValue(condition.equals, context)) + if (condition.notEquals !== undefined) return !this.equal(actual, this.resolveValue(condition.notEquals, context)) + if (condition.exists !== undefined) return (actual !== undefined) === condition.exists + return Boolean(actual) === (condition.truthy ?? true) + } + + private equal(left: unknown, right: unknown): boolean { + return isDeepStrictEqual(left, right) + } + + private commonPrefix(left: string, right: string): number { + let index = 0 + while (index < left.length && left[index] === right[index]) index += 1 + return index + } + + private referenceScopeMatches(candidateKey: string, currentKey?: string): boolean { + if (!currentKey) return true + const candidate = candidateKey.split("/") + const current = currentKey.split("/") + return current.every((segment, index) => { + if (!/\[\d+\]$/.test(segment)) return true + const candidateSegment = candidate[index] + return !candidateSegment || !/\[\d+\]$/.test(candidateSegment) || candidateSegment === segment + }) + } + + private observeUsage(info: unknown, execution: WorkflowExecutionNode, budgets: WorkflowBudget[]) { + if (!info || typeof info !== "object" + || !Object.prototype.hasOwnProperty.call(info, "cost") + || !Object.prototype.hasOwnProperty.call(info, "tokens")) return + const message = info as { cost?: number; tokens?: { total?: number; input?: number; output?: number; reasoning?: number; cache?: { read?: number; write?: number } } } + const number = (value: unknown, field: string) => { + if (value === undefined) return 0 + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Workflow SDK usage ${field} must be finite and non-negative`) + } + return value + } + if (!message.tokens || typeof message.tokens !== "object") throw new Error("Workflow SDK usage tokens must be an object") + const inputTokens = number(message.tokens.input, "tokens.input") + const outputTokens = number(message.tokens.output, "tokens.output") + const reasoningTokens = number(message.tokens.reasoning, "tokens.reasoning") + const cacheReadTokens = number(message.tokens.cache?.read, "tokens.cache.read") + const cacheWriteTokens = number(message.tokens.cache?.write, "tokens.cache.write") + const add = (left: number, right: number, field: string) => { + if (!Number.isFinite(left) || left < 0 || !Number.isFinite(right) || right < 0) { + throw new Error(`Workflow usage ${field} must be finite and non-negative`) + } + const total = left + right + if (!Number.isFinite(total)) throw new Error(`Workflow usage ${field} overflowed`) + return total + } + const usage: WorkflowUsage = { + cost: number(message.cost, "cost"), + tokens: message.tokens.total === undefined + ? [outputTokens, reasoningTokens, cacheReadTokens, cacheWriteTokens] + .reduce((total, value) => add(total, value, "tokens.total"), inputTokens) + : number(message.tokens.total, "tokens.total"), + inputTokens, + outputTokens, + reasoningTokens, + cacheReadTokens, + cacheWriteTokens, + } + const nextUsage = emptyUsage() + for (const key of Object.keys(usage) as Array) { + nextUsage[key] = add(this.run.usage![key], usage[key], key) + } + execution.usage = usage + this.run.usage = nextUsage + try { + this.enforceObservedBudget(budgets) + } catch (error) { + this.schedulerAbort.abort(error) + throw error + } + } + + private async toolOverrides(allowed: string[], signal: AbortSignal): Promise> { + const ids = await this.requireData( + this.options.client.tool.ids(undefined, { signal }), + "list workflow tools", + ) + const installed = new Set(ids) + const overrides: Record = { "*": false, ...Object.fromEntries(ids.map((id) => [id, false])) } + for (const id of allowed) { + if (!installed.has(id)) throw new Error(`Workflow agent tool ${id} is unavailable`) + overrides[id] = true + } + return overrides + } + + private operationSignal(timeoutMs: number): AbortSignal { + return AbortSignal.any([this.options.signal(timeoutMs), this.schedulerAbort.signal]) + } + + private async withActionPermit(limiters: ActionLimiter[], signal: AbortSignal, operation: () => Promise): Promise { + const acquired: ActionLimiter[] = [] + try { + for (const limiter of limiters) { + await this.acquirePermit(limiter, signal) + acquired.push(limiter) + signal.throwIfAborted() + this.throwIfCancelled() + await this.pauseIfRequested() + } + return await operation() + } finally { + for (const limiter of acquired.reverse()) this.releasePermit(limiter) + } + } + + private async acquirePermit(limiter: ActionLimiter, signal: AbortSignal): Promise { + signal.throwIfAborted() + if (limiter.active < limiter.max && limiter.waiters.length === 0) { + limiter.active += 1 + return + } + await new Promise((resolve, reject) => { + const waiter = { resolve, reject, signal, abort: () => undefined as void } + waiter.abort = () => { + const index = limiter.waiters.indexOf(waiter) + if (index >= 0) limiter.waiters.splice(index, 1) + reject(signal.reason) + } + limiter.waiters.push(waiter) + signal.addEventListener("abort", waiter.abort, { once: true }) + if (signal.aborted) waiter.abort() + }) + } + + private releasePermit(limiter: ActionLimiter): void { + while (limiter.waiters.length) { + const waiter = limiter.waiters.shift()! + waiter.signal.removeEventListener("abort", waiter.abort) + if (waiter.signal.aborted) { + waiter.reject(waiter.signal.reason) + continue + } + waiter.resolve() + return + } + limiter.active -= 1 + } + + private async sleep(delayMs: number, signal: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const abort = () => { clearTimeout(timer); reject(signal.reason) } + const timer = setTimeout(() => { signal.removeEventListener("abort", abort); resolve() }, delayMs) + signal.addEventListener("abort", abort, { once: true }) + }) + } + + private actionLimiter(max: number): ActionLimiter { + return { max, active: 0, waiters: [] } + } + + private budgetLimiter(key: string): ActionLimiter { + const existing = this.budgetLimiters.get(key) + if (existing) return existing + const limiter = this.actionLimiter(1) + this.budgetLimiters.set(key, limiter) + return limiter + } + + private sessionLimiter(key: string): ActionLimiter { + const existing = this.sessionLimiters.get(key) + if (existing) return existing + const limiter = this.actionLimiter(1) + this.sessionLimiters.set(key, limiter) + return limiter + } + + private executionDefinitionInvocationKey(execution: WorkflowExecutionNode): string { + if (execution.definitionInvocationKey) return execution.definitionInvocationKey + const segments = execution.instanceKey.split("/") + const saved = segments.map((segment, index) => ({ segment, index })) + .filter(({ segment }) => this.savedDefinitionKeys.has(segment)).at(-1) + return saved ? segments.slice(0, saved.index + 1).join("/") : `${this.run.definitionId}@${this.run.definitionRevision}` + } + + private forgetSession(execution: WorkflowExecutionNode, sessionId: string): void { + if (execution.sessionIds) { + execution.sessionIds = execution.sessionIds.filter((candidate) => candidate !== sessionId) + if (execution.sessionIds.length === 0) delete execution.sessionIds + } + if (!this.run.sessionBindings) return + for (const [key, binding] of Object.entries(this.run.sessionBindings)) { + if (binding === sessionId) delete this.run.sessionBindings[key] + } + if (Object.keys(this.run.sessionBindings).length === 0) delete this.run.sessionBindings + } + + private async persistSessionCleanup(execution: WorkflowExecutionNode, cause: unknown): Promise { + execution.status = "waiting" + delete execution.error + delete execution.completedAt + try { + await this.options.persist() + } catch (error) { + throw new WorkflowRetryCleanupError(`Workflow session cleanup could not be persisted: ${this.errorMessage(cause)}; ${this.errorMessage(error)}`) + } + } + + private enforceActionAdmission(budgets: WorkflowBudget[]) { + for (const budget of budgets) { + if (budget.maxCost !== undefined && this.run.usage!.cost >= budget.maxCost) { + throw new WorkflowBudgetError(`Workflow reached cost budget ${budget.maxCost}`) + } + if (budget.maxTokens !== undefined && this.run.usage!.tokens >= budget.maxTokens) { + throw new WorkflowBudgetError(`Workflow reached token budget ${budget.maxTokens}`) + } + } + } + + private enforceObservedBudget(budgets: WorkflowBudget[]) { + // ponytail: providers report usage after an action, so one admitted action can overshoot; no estimate is invented. + for (const budget of budgets) { + if (budget.maxCost !== undefined && this.run.usage!.cost > budget.maxCost) { + throw new WorkflowBudgetError(`Workflow exceeded cost budget ${budget.maxCost}`) + } + if (budget.maxTokens !== undefined && this.run.usage!.tokens > budget.maxTokens) { + throw new WorkflowBudgetError(`Workflow exceeded token budget ${budget.maxTokens}`) + } + } + } + + private async pauseIfRequested() { + if (!this.run.pauseRequested || this.options.isPauseCommitted?.() === false) return + throw new WorkflowSuspendedError() + } + + private throwIfCancelled() { + if (this.options.isCancelled()) throw new Error("Workflow run cancelled") + } + + private boundOutput(output: unknown, structural = false): { output: unknown; truncated: boolean } { + if (output === undefined) return { output: undefined, truncated: false } + if (structural) return { output, truncated: false } + if (typeof output === "string") return output.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: output.slice(0, MAX_OUTPUT_CHARS), truncated: true } + const serialized = JSON.stringify(output) + return serialized.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: serialized.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + + private sessionMetadata(role: string) { + return { codenomad: { version: 1, workflow: { runId: this.run.id, role } } } + } + + private async requireData(request: Promise<{ data?: T; error?: unknown }>, action: string): Promise { + const response = await request + if (response.data !== undefined) return response.data + throw new Error(`${action} failed: ${this.errorMessage(response.error)}`) + } + + private errorMessage(error: unknown): string { + if (error instanceof Error) return error.message + if (typeof error === "string") return error + try { return JSON.stringify(error) || "Unknown error" } catch { return "Unknown error" } + } +} diff --git a/packages/server/src/workflows/json-schema.ts b/packages/server/src/workflows/json-schema.ts new file mode 100644 index 000000000..d2d0c9225 --- /dev/null +++ b/packages/server/src/workflows/json-schema.ts @@ -0,0 +1,168 @@ +const JSON_TYPES = new Set(["null", "array", "object", "integer", "number", "string", "boolean"]) +const SCHEMA_KEYS = new Set([ + "type", "enum", "const", "minLength", "maxLength", "minimum", "maximum", + "minItems", "maxItems", "items", "required", "properties", "additionalProperties", + "allOf", "anyOf", "oneOf", +]) +const SCHEMA_DEPTH_LIMIT = 20 +const SCHEMA_NODE_LIMIT = 2_000 + +export interface JsonSchemaIssue { + path: Array + message: string +} + +const isRecord = (value: unknown): value is Record => + Boolean(value) && typeof value === "object" && !Array.isArray(value) + +const nonNegativeInteger = (value: unknown) => Number.isInteger(value) && (value as number) >= 0 + +const jsonEqual = (left: unknown, right: unknown): boolean => { + if (left === right) return true + if (Array.isArray(left) || Array.isArray(right)) { + return Array.isArray(left) && Array.isArray(right) && left.length === right.length + && left.every((value, index) => jsonEqual(value, right[index])) + } + if (!isRecord(left) || !isRecord(right)) return false + const keys = Object.keys(left) + return keys.length === Object.keys(right).length + && keys.every((key) => Object.prototype.hasOwnProperty.call(right, key) && jsonEqual(left[key], right[key])) +} + +export function inspectJsonSchema(schema: Record): JsonSchemaIssue[] { + const issues: JsonSchemaIssue[] = [] + const pending: Array<{ schema: Record; path: Array; depth: number }> = [ + { schema, path: [], depth: 0 }, + ] + let nodes = 0 + + while (pending.length) { + const current = pending.pop()! + if (++nodes > SCHEMA_NODE_LIMIT) { + issues.push({ path: current.path, message: "JSON schema has too many nested schemas" }) + break + } + if (current.depth > SCHEMA_DEPTH_LIMIT) { + issues.push({ path: current.path, message: "JSON schema is too deeply nested" }) + continue + } + const add = (key: string, message: string) => issues.push({ path: [...current.path, key], message }) + + for (const key of Object.keys(current.schema)) { + if (!SCHEMA_KEYS.has(key)) add(key, `Unsupported JSON schema keyword: ${key}`) + } + + if (current.schema.type !== undefined) { + const types = typeof current.schema.type === "string" ? [current.schema.type] : current.schema.type + if (!Array.isArray(types) || types.length === 0 || types.some((type) => typeof type !== "string" || !JSON_TYPES.has(type))) { + add("type", "JSON schema type must contain only supported JSON types") + } else if (new Set(types).size !== types.length) { + add("type", "JSON schema types must be unique") + } + } + if (current.schema.enum !== undefined) { + if (!Array.isArray(current.schema.enum)) add("enum", "JSON schema enum must be an array") + else if (current.schema.enum.length === 0) add("enum", "JSON schema enum must not be empty") + else if (current.schema.enum.some((value, index, values) => values.slice(0, index).some((other) => jsonEqual(value, other)))) { + add("enum", "JSON schema enum values must be unique") + } + } + for (const key of ["minLength", "maxLength", "minItems", "maxItems"] as const) { + if (current.schema[key] !== undefined && !nonNegativeInteger(current.schema[key])) add(key, `${key} must be a non-negative integer`) + } + for (const key of ["minimum", "maximum"] as const) { + if (current.schema[key] !== undefined && (typeof current.schema[key] !== "number" || !Number.isFinite(current.schema[key]))) { + add(key, `${key} must be a finite number`) + } + } + if (current.schema.required !== undefined) { + const required = current.schema.required + if (!Array.isArray(required) || required.some((key) => typeof key !== "string")) add("required", "required must be an array of property names") + else if (new Set(required).size !== required.length) add("required", "required property names must be unique") + } + if (current.schema.additionalProperties !== undefined && typeof current.schema.additionalProperties !== "boolean") { + add("additionalProperties", "additionalProperties must be a boolean") + } + + const enqueue = (value: unknown, path: Array) => { + if (!isRecord(value)) { + issues.push({ path, message: "Nested JSON schema must be an object" }) + return + } + pending.push({ schema: value, path, depth: current.depth + 1 }) + } + if (current.schema.items !== undefined) enqueue(current.schema.items, [...current.path, "items"]) + if (current.schema.properties !== undefined) { + if (!isRecord(current.schema.properties)) add("properties", "properties must be an object") + else for (const [key, child] of Object.entries(current.schema.properties)) enqueue(child, [...current.path, "properties", key]) + } + for (const key of ["allOf", "anyOf", "oneOf"] as const) { + const alternatives = current.schema[key] + if (alternatives === undefined) continue + if (!Array.isArray(alternatives) || alternatives.length === 0) { + add(key, `${key} must be a non-empty array of schemas`) + continue + } + alternatives.forEach((child, index) => enqueue(child, [...current.path, key, index])) + } + } + return issues +} + +const typeMatches = (value: unknown, type: string) => { + if (type === "null") return value === null + if (type === "array") return Array.isArray(value) + if (type === "object") return Boolean(value) && typeof value === "object" && !Array.isArray(value) + if (type === "integer") return Number.isInteger(value) + if (type === "number") return typeof value === "number" && Number.isFinite(value) + return typeof value === type +} + +export function validateJsonSchemaValue(value: unknown, schema: Record, path = "$"): string[] { + const errors: string[] = [] + const types = typeof schema.type === "string" ? [schema.type] : Array.isArray(schema.type) ? schema.type : [] + if (types.length && !types.some((type) => typeof type === "string" && typeMatches(value, type))) { + return [`${path} must be ${types.join(" or ")}`] + } + if (Array.isArray(schema.enum) && !schema.enum.some((candidate) => jsonEqual(candidate, value))) errors.push(`${path} is not an allowed value`) + if (Object.prototype.hasOwnProperty.call(schema, "const") && !jsonEqual(schema.const, value)) errors.push(`${path} must equal const`) + + if (typeof value === "string") { + const length = [...value].length + if (typeof schema.minLength === "number" && length < schema.minLength) errors.push(`${path} is too short`) + if (typeof schema.maxLength === "number" && length > schema.maxLength) errors.push(`${path} is too long`) + } + if (typeof value === "number") { + if (typeof schema.minimum === "number" && value < schema.minimum) errors.push(`${path} is below minimum`) + if (typeof schema.maximum === "number" && value > schema.maximum) errors.push(`${path} is above maximum`) + } + if (Array.isArray(value)) { + if (typeof schema.minItems === "number" && value.length < schema.minItems) errors.push(`${path} has too few items`) + if (typeof schema.maxItems === "number" && value.length > schema.maxItems) errors.push(`${path} has too many items`) + if (isRecord(schema.items)) value.forEach((item, index) => errors.push(...validateJsonSchemaValue(item, schema.items as Record, `${path}[${index}]`))) + } + if (isRecord(value)) { + if (Array.isArray(schema.required)) for (const key of schema.required) { + if (typeof key === "string" && !Object.prototype.hasOwnProperty.call(value, key)) errors.push(`${path}.${key} is required`) + } + const properties = isRecord(schema.properties) ? schema.properties : {} + for (const [key, childSchema] of Object.entries(properties)) { + if (Object.prototype.hasOwnProperty.call(value, key) && isRecord(childSchema)) { + errors.push(...validateJsonSchemaValue(value[key], childSchema, `${path}.${key}`)) + } + } + if (schema.additionalProperties === false) { + for (const key of Object.keys(value)) if (!Object.prototype.hasOwnProperty.call(properties, key)) errors.push(`${path}.${key} is not allowed`) + } + } + + for (const keyword of ["allOf", "anyOf", "oneOf"] as const) { + if (!Array.isArray(schema[keyword])) continue + const results = schema[keyword].filter(isRecord).map((item) => validateJsonSchemaValue(value, item, path)) + const matches = results.filter((result) => result.length === 0).length + if (keyword === "allOf" && matches !== results.length) errors.push(`${path} does not match allOf`) + if (keyword === "anyOf" && matches === 0) errors.push(`${path} does not match anyOf`) + if (keyword === "oneOf" && matches !== 1) errors.push(`${path} does not match exactly one oneOf schema`) + } + return errors +} diff --git a/packages/server/src/workflows/manager.test.ts b/packages/server/src/workflows/manager.test.ts new file mode 100644 index 000000000..38d83f989 --- /dev/null +++ b/packages/server/src/workflows/manager.test.ts @@ -0,0 +1,1013 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { describe, it } from "node:test" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import type { WorkspaceManager } from "../workspaces/manager" +import { withFilesystemLock } from "./filesystem-lock" +import { WorkflowManager, WorkflowRunError } from "./manager" + +describe("WorkflowManager", () => { + it("keeps a live executor leased when a second manager starts", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-live-owner-")) + let releasePrompt!: () => void + let promptStarted!: () => void + const startedPrompt = new Promise((resolve) => { promptStarted = resolve }) + const blockedPrompt = new Promise((resolve) => { releasePrompt = resolve }) + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: "live-session" } }), + prompt: async () => { promptStarted(); await blockedPrompt; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "live-lineage", path: "C:/live-workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "live-lineage", path: "C:/live-workspace", status: "ready" }], + } as unknown as WorkspaceManager + const options = { workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client } + const owner = new WorkflowManager(options) + let observer: WorkflowManager | undefined + try { + await owner.createDefinition({ version: 1, id: "live", name: "Live", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const started = await owner.start({ workspaceId: "workspace", definitionId: "live" }) + await startedPrompt + const before = JSON.parse(await fs.readFile(path.join(storageDir, `${started.id}.json`), "utf8")) + assert.ok(Date.parse(before.executorLease.expiresAt) > Date.now()) + + observer = new WorkflowManager(options) + assert.equal((await observer.get(started.id))?.status, "running") + const after = JSON.parse(await fs.readFile(path.join(storageDir, `${started.id}.json`), "utf8")) + assert.equal(after.executorLease.ownerToken, before.executorLease.ownerToken) + assert.equal(after.executorLease.fence, before.executorLease.fence) + } finally { + releasePrompt() + await owner.shutdown() + await observer?.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("never reclaims a same-process executor merely for missed heartbeats", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-remote-cancel-")) + let releasePrompt!: () => void + let promptStarted!: () => void + const startedPrompt = new Promise((resolve) => { promptStarted = resolve }) + const blockedPrompt = new Promise((resolve) => { releasePrompt = resolve }) + let aborts = 0 + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: "remote-session" } }), + prompt: async () => { promptStarted(); await blockedPrompt; return { data: { info: {}, parts: [] } } }, + abort: async () => { aborts++; return { data: true } }, + } } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "remote-lineage", path: "C:/remote-workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "remote-lineage", path: "C:/remote-workspace", status: "ready" }], + } as unknown as WorkspaceManager + const options = { workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client } + const owner = new WorkflowManager(options) + const remote = new WorkflowManager(options) + try { + await owner.createDefinition({ version: 1, id: "remote", name: "Remote", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const started = await owner.start({ workspaceId: "workspace", definitionId: "remote" }) + await startedPrompt + await assert.rejects(remote.pause(started.id), (error: WorkflowRunError) => error.statusCode === 409) + await assert.rejects(remote.cancel(started.id), (error: WorkflowRunError) => error.statusCode === 409) + await assert.rejects(remote.resume(started.id), (error: WorkflowRunError) => error.statusCode === 409) + assert.equal(aborts, 0) + assert.equal((await remote.get(started.id))?.status, "running") + + const journalPath = path.join(storageDir, `${started.id}.json`) + const expired = JSON.parse(await fs.readFile(journalPath, "utf8")) + expired.executorLease.heartbeatAt = new Date(Date.now() - 80_000).toISOString() + expired.executorLease.expiresAt = new Date(Date.now() - 70_000).toISOString() + await fs.writeFile(journalPath, `${JSON.stringify(expired, null, 2)}\n`, "utf8") + await assert.rejects(remote.pause(started.id), (error: WorkflowRunError) => error.statusCode === 409) + await assert.rejects(remote.cancel(started.id), (error: WorkflowRunError) => error.statusCode === 409) + assert.equal(JSON.parse(await fs.readFile(journalPath, "utf8")).status, "running") + assert.equal(aborts, 0) + releasePrompt() + for (let attempt = 0; attempt < 100 && JSON.parse(await fs.readFile(journalPath, "utf8")).status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal(JSON.parse(await fs.readFile(journalPath, "utf8")).status, "completed") + } finally { + releasePrompt() + await owner.shutdown() + await remote.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("revalidates the durable executor fence immediately before prompting", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-action-fence-")) + let prompts = 0 + let sessions = 0 + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: `fence-session-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "fence-lineage", path: "C:/fence-workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "fence-lineage", path: "C:/fence-workspace", status: "ready" }], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client }) + let fenceEntered!: () => void + let releaseFence!: () => void + const entered = new Promise((resolve) => { fenceEntered = resolve }) + const blocked = new Promise((resolve) => { releaseFence = resolve }) + const revalidate = (manager as any).revalidateExecutorFence.bind(manager) + ;(manager as any).revalidateExecutorFence = async (active: unknown) => { + fenceEntered() + await blocked + return revalidate(active) + } + try { + await manager.createDefinition({ version: 1, id: "fenced", name: "Fenced", root: { + type: "agent", id: "action", instructions: "Must not start", + } }) + const run = await manager.start({ workspaceId: "workspace", definitionId: "fenced" }) + await entered + const journalPath = path.join(storageDir, `${run.id}.json`) + const replaced = JSON.parse(await fs.readFile(journalPath, "utf8")) + replaced.executorLease = { + ownerToken: "successor", fence: replaced.executorFence + 1, + heartbeatAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), + } + replaced.executorFence += 1 + replaced.revision += 1 + await fs.writeFile(journalPath, `${JSON.stringify(replaced, null, 2)}\n`, "utf8") + releaseFence() + for (let attempt = 0; attempt < 100 && !(manager as any).activeRuns.get(run.id)?.leaseLost; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal(prompts, 0) + assert.equal((manager as any).activeRuns.get(run.id)?.leaseLost, true) + } finally { + releaseFence?.() + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("refreshes ownership created and released by another manager under admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-refresh-owner-")) + let releasePrompt!: () => void + let promptStarted!: () => void + const startedPrompt = new Promise((resolve) => { promptStarted = resolve }) + const blockedPrompt = new Promise((resolve) => { releasePrompt = resolve }) + const client = { session: { + create: async () => ({ data: { id: "ownership-session" } }), + prompt: async () => { promptStarted(); await blockedPrompt; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "ownership-lineage", path: "C:/ownership-workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "ownership-lineage", path: "C:/ownership-workspace", status: "ready" }], + } as unknown as WorkspaceManager + const options = { workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client } + const observer = new WorkflowManager(options) + const owner = new WorkflowManager(options) + try { + await observer.list() + const started = await owner.start({ workspaceId: "workspace", objective: "Own", stages: [ + { id: "work", title: "Work", instructions: "Wait" }, + ] }) + await startedPrompt + assert.equal(await observer.withWorkspaceOwnershipLease( + { lineageId: "ownership-lineage" }, async (owned) => owned, + ), true) + releasePrompt() + for (let attempt = 0; attempt < 100 && (await owner.get(started.id))?.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal(await observer.withWorkspaceOwnershipLease( + { lineageId: "ownership-lineage" }, async (owned) => owned, + ), false) + } finally { + releasePrompt() + await owner.shutdown() + await observer.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("coalesces read refreshes and recovers an executor after its process dies", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-read-recovery-")) + const manager = new WorkflowManager({ + workspaceManager: { get: () => undefined, list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + const id = "00000000-0000-4000-8000-000000000011" + try { + await manager.list() + const now = new Date().toISOString() + await fs.writeFile(path.join(storageDir, `${id}.json`), JSON.stringify({ + id, workspaceId: "dead-workspace", workspaceLineageId: "dead-lineage", workspacePath: "C:/dead-workspace", + objective: "Recover dead executor", status: "running", revision: 1, + steps: [{ id: "pending", title: "Pending", instructions: "Wait", status: "pending" }], + executorFence: 1, + executorLease: { + ownerToken: "dead-owner", fence: 1, hostname: os.hostname(), pid: 2_147_483_647, processStart: "dead", + heartbeatAt: new Date(Date.now() - 20_000).toISOString(), + expiresAt: new Date(Date.now() - 10_000).toISOString(), + }, + createdAt: now, updatedAt: now, + }), "utf8") + let recoveries = 0 + const recover = (manager as any).recoverInterruptedRuns.bind(manager) + ;(manager as any).recoverInterruptedRuns = async () => { recoveries++; return recover() } + ;(manager as any).lastReadRefreshAt = 0 + + const [got, listed, gotAgain] = await Promise.all([manager.get(id), manager.list(), manager.get(id)]) + assert.equal(recoveries, 1) + assert.equal(got?.status, "interrupted") + assert.equal(gotAgain?.status, "interrupted") + assert.equal(listed[0]?.status, "interrupted") + assert.equal(got?.executorLease, undefined) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("uses a caller-known run ID idempotently without overwriting another start", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-run-id-")) + let releasePrompt!: () => void + const blocked = new Promise((resolve) => { releasePrompt = resolve }) + let sessions = 0 + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: `idempotent-${++sessions}` } }), + prompt: async () => { await blocked; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ + workspaceManager: { get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }) } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, createClient: () => client, + }) + const runId = "00000000-0000-4000-8000-000000000012" + const request = { runId, workspaceId: "workspace", definitionId: "known-id", objective: "Ship", inputs: { target: "test" } } + try { + await manager.createDefinition({ version: 1, id: "known-id", name: "Known ID", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const [first, retry] = await Promise.all([manager.start(request), manager.start(request)]) + assert.equal(first.id, runId) + assert.equal(retry.id, runId) + assert.equal(first.createdAt, retry.createdAt) + await assert.rejects(manager.start({ ...request, objective: "Different" }), (error: WorkflowRunError) => + error.statusCode === 409 && /another start request/.test(error.message)) + assert.equal((await manager.get(runId))?.objective, "Ship") + } finally { + releasePrompt() + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("retries initialization after timing out behind a long admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-init-retry-")) + let releaseAdmission!: () => void + let admissionEntered!: () => void + const entered = new Promise((resolve) => { admissionEntered = resolve }) + const held = new Promise((resolve) => { releaseAdmission = resolve }) + const admission = withFilesystemLock(path.join(storageDir, ".admission.lock"), async () => { + admissionEntered() + await held + }) + await entered + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + try { + await assert.rejects(manager.list(), /Timed out waiting for filesystem lock/) + releaseAdmission() + await admission + await assert.doesNotReject(manager.list()) + } finally { + releaseAdmission() + await admission + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("releases admission while cancelling a prompt that ignores abort", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-cancel-admission-")) + let promptStarted!: () => void + let releasePrompt!: () => void + const started = new Promise((resolve) => { promptStarted = resolve }) + const blockedPrompt = new Promise((resolve) => { releasePrompt = resolve }) + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: "ignored-abort-session" } }), + prompt: async () => { promptStarted(); await blockedPrompt; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ id, lineageId: `lineage-${id}`, path: `C:/${id}`, status: "ready" }), + list: () => [], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ + workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client, + promptTimeoutMs: 500, + }) + try { + await manager.createDefinition({ version: 1, id: "blocked", name: "Blocked", root: { + type: "agent", id: "work", instructions: "Ignore abort", + } }) + const run = await manager.start({ workspaceId: "blocked", definitionId: "blocked" }) + await started + const cancellation = manager.cancel(run.id) + let cancellationSettled = false + void cancellation.then(() => { cancellationSettled = true }) + let cancellationFenced = false + for (let attempt = 0; attempt < 100; attempt += 1) { + const persisted = JSON.parse(await fs.readFile(path.join(storageDir, `${run.id}.json`), "utf8")) + if (persisted.status === "recovery_required") { + cancellationFenced = true + break + } + await new Promise((resolve) => setTimeout(resolve, 1)) + } + assert.equal(cancellationFenced, true) + assert.equal(cancellationSettled, false) + + const definition = await manager.createDefinition({ version: 1, id: "admitted", name: "Admitted", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Wait", + } }) + const other = await manager.start({ workspaceId: "other", definitionId: definition.id }) + assert.equal((await manager.cancel(other.id))?.status, "cancelled") + assert.equal((await cancellation)?.status, "cancelled") + releasePrompt() + } finally { + releasePrompt?.() + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("keeps confirmed resume settlement outside admission while cancellation is fenced", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-resume-cancel-admission-")) + let promptStarted!: () => void + let releasePrompt!: () => void + const started = new Promise((resolve) => { promptStarted = resolve }) + const blockedPrompt = new Promise((resolve) => { releasePrompt = resolve }) + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: "resume-cancel-session" } }), + prompt: async () => { promptStarted(); await blockedPrompt; return { data: { info: {}, parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ id, lineageId: `lineage-${id}`, path: `C:/${id}`, status: "ready" }), + list: () => [], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ + workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client, + promptTimeoutMs: 1_000, + }) + try { + await manager.createDefinition({ version: 1, id: "resume-cancel", name: "Resume cancel", root: { + type: "agent", id: "work", instructions: "Ignore abort", + } }) + const run = await manager.start({ workspaceId: "blocked", definitionId: "resume-cancel" }) + await started + const cancellation = manager.cancel(run.id) + let recovery: any + for (let attempt = 0; attempt < 100; attempt += 1) { + recovery = JSON.parse(await fs.readFile(path.join(storageDir, `${run.id}.json`), "utf8")) + if (recovery.status === "recovery_required") break + await new Promise((resolve) => setTimeout(resolve, 1)) + } + assert.equal(recovery.status, "recovery_required") + + const resume = manager.resume(run.id, true, recovery.revision) + const admitted = await Promise.race([ + manager.createDefinition({ version: 1, id: "unrelated-admission", name: "Unrelated", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Wait", + } }), + new Promise((_, reject) => setTimeout(() => reject(new Error("unrelated admission was blocked by resume")), 500)), + ]) + assert.equal(admitted.id, "unrelated-admission") + assert.equal((await cancellation)?.status, "cancelled") + assert.ok(["cancelled", "recovery_required"].includes((await resume)?.status ?? "")) + releasePrompt() + } finally { + releasePrompt?.() + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("does not deadlock a control transition against cancellation", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-control-cancel-")) + let sessions = 0 + const client = { tool: { ids: async () => ({ data: [] }) }, session: { + create: async () => ({ data: { id: `control-cancel-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ id, lineageId: `lineage-${id}`, path: `C:/${id}`, status: "ready" }), + list: () => [], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ + workspaceManager, eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, storageDir, createClient: () => client, + }) + let releaseAdmission!: () => void + let admissionEntered!: () => void + const entered = new Promise((resolve) => { admissionEntered = resolve }) + const blocked = new Promise((resolve) => { releaseAdmission = resolve }) + const originalRefresh = (manager as any).refreshRunIndex.bind(manager) + try { + await manager.createDefinition({ version: 1, id: "control-cancel", name: "Control cancel", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Wait", + } }) + const started = await manager.start({ workspaceId: "controlled", definitionId: "control-cancel" }) + let waiting = await manager.get(started.id) + while (!waiting?.pendingGate) { + await new Promise((resolve) => setTimeout(resolve, 1)) + waiting = await manager.get(started.id) + } + const gateId = waiting.pendingGate.executionNodeId + + let blockNextRefresh = true + ;(manager as any).refreshRunIndex = async () => { + await originalRefresh() + if (!blockNextRefresh) return + blockNextRefresh = false + admissionEntered() + await blocked + } + const answer = manager.answer(started.id, gateId, true) + await entered + const cancellation = manager.cancel(started.id) + await new Promise((resolve) => setImmediate(resolve)) + releaseAdmission() + + await Promise.race([ + Promise.allSettled([answer, cancellation]), + new Promise((_, reject) => setTimeout(() => reject(new Error("control and cancellation deadlocked")), 1_000)), + ]) + const other = await Promise.race([ + manager.start({ workspaceId: "unrelated", definitionId: "control-cancel" }), + new Promise((_, reject) => setTimeout(() => reject(new Error("unrelated start was not admitted")), 1_000)), + ]) + assert.equal((await manager.cancel(other.id))?.status, "cancelled") + } finally { + releaseAdmission?.() + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("admits one run per path or lineage across manager instances", async () => { + for (const collision of ["lineage", "path"] as const) { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), `codenomad-workflow-shared-${collision}-`)) + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `shared-${++sessions}` } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Review" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ + id, + lineageId: collision === "lineage" ? "shared-lineage" : `lineage-${id}`, + path: collision === "path" ? "C:/shared-workspace" : `C:/${id}`, + status: "ready", + }), + list: () => [], + } as unknown as WorkspaceManager + const options = { + workspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + createClient: () => client, + } + const first = new WorkflowManager(options) + const second = new WorkflowManager(options) + try { + await Promise.all([first.list(), second.list()]) + const starts = await Promise.allSettled([ + first.start({ workspaceId: "first", objective: "First", stages: [ + { id: "work", title: "Work", instructions: "Work", requiresApproval: true }, + ] }), + second.start({ workspaceId: "second", objective: "Second", stages: [ + { id: "work", title: "Work", instructions: "Work", requiresApproval: true }, + ] }), + ]) + assert.equal(starts.filter((result) => result.status === "fulfilled").length, 1) + assert.match((starts.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /already running/) + } finally { + await Promise.allSettled([first.shutdown(), second.shutdown()]) + await fs.rm(storageDir, { recursive: true, force: true }) + } + } + }) + + it("rejects a stale saved definition inside the admission queue", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-latest-")) + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + createClient: () => null, + }) + try { + const definition = { version: 1 as const, id: "latest", name: "Latest", root: { + type: "agent" as const, id: "work", instructions: "Work", + } } + await manager.createDefinition(definition) + await manager.listDefinitions() + const [updated, stale] = await Promise.allSettled([ + manager.updateDefinition("latest", 1, { ...definition, name: "Updated" }), + manager.startLatest({ workspaceId: "workspace", definitionId: "latest", definitionRevision: 1 }), + ]) + assert.equal(updated.status, "fulfilled") + assert.equal(stale.status, "rejected") + assert.match((stale as PromiseRejectedResult).reason.message, /revision is stale/) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("resolves an omitted latest revision inside admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-current-")) + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `latest-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ + workspaceManager: { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + createClient: () => client, + }) + try { + const definition = { version: 1 as const, id: "current", name: "Current", root: { + type: "gate" as const, id: "gate", gate: "approval" as const, prompt: "Wait", + } } + await manager.createDefinition(definition) + const [updated, started] = await Promise.all([ + manager.updateDefinition("current", 1, { ...definition, name: "Updated" }), + manager.startLatest({ workspaceId: "workspace", definitionId: "current" }), + ]) + assert.equal(updated.revision, 2) + assert.equal(started.definitionRevision, 2) + assert.equal(started.definitionSnapshot?.name, "Updated") + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("retains creation cleanup after three transient failures and retries it on later admission", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-cleanup-")) + let attempts = 0 + const manager = new WorkflowManager({ + workspaceManager: { + list: () => [], + cancelCreationRequest: async () => { + if (++attempts <= 3) throw new Error("transient cleanup failure") + }, + } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + try { + ;(manager as any).deferCreationCleanup("request") + for (let attempt = 0; attempt < 3; attempt += 1) await (manager as any).drainCreationCleanups() + assert.equal(attempts, 3) + assert.equal((manager as any).deferredCreationCleanups.has("request"), true) + await assert.rejects(manager.startLatest({ workspaceId: "workspace", definitionId: "missing" }), /not found/) + assert.equal(attempts, 4) + assert.equal((manager as any).deferredCreationCleanups.size, 0) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("keeps shutdown retryable while deferred creation cleanup remains", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-cleanup-shutdown-")) + let attempts = 0 + const manager = new WorkflowManager({ + workspaceManager: { + list: () => [], + cancelCreationRequest: async () => { + if (++attempts <= 3) throw new Error("transient cleanup failure") + }, + } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + try { + ;(manager as any).deferCreationCleanup("request") + await assert.rejects(manager.shutdown(), /creation cleanup remains pending/) + assert.equal((manager as any).deferredCreationCleanups.has("request"), true) + await manager.shutdown() + assert.equal(attempts, 4) + assert.equal((manager as any).deferredCreationCleanups.size, 0) + } finally { + await manager.shutdown().catch(() => undefined) + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("cancels a legacy approval that persists while shutdown starts", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-approval-shutdown-")) + const client = { session: { + create: async () => ({ data: { id: "legacy-session" } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Review" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + let manager: WorkflowManager + let armShutdown = false + let shutdown: Promise | undefined + const eventBus = { publish: (event: any) => { + if (armShutdown && event.event?.properties?.status === "running") { + armShutdown = false + shutdown = manager.shutdown() + } + return true + } } as unknown as EventBus + manager = new WorkflowManager({ workspaceManager, eventBus, logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, createClient: () => client }) + try { + const started = await manager.start({ workspaceId: "workspace", objective: "Legacy", stages: [ + { id: "review", title: "Review", instructions: "Review", requiresApproval: true }, + ] }) + let waiting = await manager.get(started.id) + for (let attempt = 0; attempt < 200 && waiting?.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + waiting = await manager.get(started.id) + } + assert.equal(waiting?.status, "waiting_for_review") + + armShutdown = true + const approved = await manager.approve(started.id, "review") + await shutdown + assert.equal(approved?.status, "cancelled") + assert.equal(JSON.parse(await fs.readFile(path.join(storageDir, `${started.id}.json`), "utf8")).status, "cancelled") + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("quarantines managed-worktree ownership from a malformed active journal", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-worktree-quarantine-")) + await fs.writeFile(path.join(storageDir, "malformed.json"), JSON.stringify({ + id: "malformed", status: "running", workspaceId: "execution", workspaceLineageId: "execution-lineage", + workspacePath: "C:/repo/.codenomad/worktrees/review", + worktreeSelection: { + sourceWorkspaceId: "source", sourceWorkspaceLineageId: "source-lineage", sourceWorkspacePath: "C:/repo", + directory: "C:/repo/.codenomad/worktrees/review", slug: "review", + }, + }), "utf8") + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + try { + assert.equal(await manager.isWorktreeWorkflowOwned( + { lineageId: "source-lineage" }, { slug: "review" }, + ), true) + assert.equal(await manager.isWorktreeWorkflowOwned( + { lineageId: "unrelated" }, { path: "C:/repo/.codenomad/worktrees/review" }, + ), true) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("persists and hands a structured planner result to the implementer", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflows-")) + const creates: Array | undefined> = [] + const prompts: Array> = [] + let session = 0 + const client = { + session: { + create: async (input?: Record) => { + creates.push(input) + return { data: { id: `session-${++session}` } } + }, + prompt: async (input: Record) => { + prompts.push(input) + if (prompts.length === 1) { + return { + data: { + info: { structured: { summary: "Plan", steps: ["Change code", "Run test"] } }, + parts: [], + }, + } + } + const text = prompts.length === 2 ? "Reviewed plan" : "Implemented" + return { data: { info: {}, parts: [{ type: "text", text }] } } + }, + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const events: unknown[] = [] + const eventBus = { publish: (event: unknown) => { events.push(event); return true } } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + }) + let reloaded: WorkflowManager | undefined + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Add workflow support", + stages: [ + { id: "planner", title: "Planner", instructions: "Create a plan", requiresApproval: true }, + { id: "reviewer", title: "Reviewer", instructions: "Review the approved plan", requiresApproval: true }, + { id: "implementer", title: "Implementer", instructions: "Implement the reviewed plan", requiresApproval: true }, + ], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.equal(run.rootSessionId, "session-1") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "pending", undefined], + ["implementer", "pending", undefined], + ]) + + run = (await manager.approve(started.id, "planner"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "completed", "session-3"], + ["implementer", "pending", undefined], + ]) + + run = (await manager.approve(started.id, "reviewer"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(run.steps.map((step) => [step.id, step.status, step.sessionId]), [ + ["planner", "completed", "session-2"], + ["reviewer", "completed", "session-3"], + ["implementer", "completed", "session-4"], + ]) + assert.equal(creates[1]?.parentID, "session-1") + assert.equal(creates[2]?.parentID, "session-1") + assert.match(JSON.stringify(prompts[1]), /Change code/) + assert.match(JSON.stringify(prompts[2]), /Reviewed plan/) + assert.ok(events.length >= 10) + + await manager.shutdown() + const restoredWorkspaceManager = { + get: (id: string) => id === "workspace-restored" + ? { id, lineageId: "lineage-a", path: "C:/workspace", status: "ready" } + : undefined, + list: () => [{ id: "workspace-restored", lineageId: "lineage-a", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + reloaded = new WorkflowManager({ + workspaceManager: restoredWorkspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + }) + await assert.rejects( + reloaded.start({ + workspaceId: "workspace-restored", + objective: "Conflicting run", + stages: [{ id: "other", title: "Other", instructions: "Do other work" }], + }), + /workspace lineage/, + ) + run = (await reloaded.approve(started.id, "implementer"))! + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await reloaded.get(started.id))! + } + assert.equal(run.status, "completed") + const [restored] = await reloaded.list("workspace-restored") + assert.equal(restored?.workspaceId, "workspace-restored") + } finally { + await manager.shutdown() + await reloaded?.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("fails a stage when its OpenCode request exceeds the operation timeout", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-timeout-")) + let session = 0 + let aborts = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => new Promise((_, reject) => { + const signal = options?.signal + if (!signal) return + signal.addEventListener("abort", () => reject(signal.reason), { once: true }) + }), + abort: async () => ++aborts === 1 ? { error: "abort failed" } : { data: true }, + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage-timeout", path: "C:/timeout-workspace", status: "ready" }), + } as unknown as WorkspaceManager + const eventBus = { publish: () => true } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus, + logger, + storageDir, + createClient: () => client, + promptTimeoutMs: 10, + }) + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Never finish", + stages: [{ id: "blocked", title: "Blocked", instructions: "Wait forever" }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + assert.equal(run.status, "recovery_required") + assert.equal(run.steps[0]?.status, "failed") + assert.equal(aborts, 1) + await assert.rejects(manager.start({ + workspaceId: "workspace", + objective: "Must remain blocked", + stages: [{ id: "next", title: "Next", instructions: "Do not start" }], + }), /already running/) + assert.equal((await manager.cancel(started.id))?.status, "cancelled") + assert.equal(aborts, 2) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("keeps force-created instances of the same path isolated by lineage", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-lineage-")) + let session = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Review me" }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: (id: string) => ({ id, lineageId: id === "a" ? "lineage-a" : "lineage-b", path: "C:/same", status: "ready" }), + } as unknown as WorkspaceManager + const eventBus = { publish: () => true } as unknown as EventBus + const logger = { warn() {}, error() {} } as unknown as Logger + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir, createClient: () => client }) + + try { + const started = await manager.start({ + workspaceId: "a", + objective: "Lineage A", + stages: [{ id: "only", title: "Only", instructions: "Run", requiresApproval: true }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + assert.equal(run.status, "waiting_for_review") + assert.deepEqual(await manager.list("b"), []) + assert.equal(await manager.get(started.id, "b"), undefined) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("does not fail a completed run when history pruning fails", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-prune-")) + let session = 0 + const client = { + session: { + create: async () => ({ data: { id: `session-${++session}` } }), + prompt: async () => ({ data: { info: {}, parts: [{ type: "text", text: "Complete" }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const workspaceManager = { + get: () => ({ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }), + } as unknown as WorkspaceManager + let warnings = 0 + const logger = { warn: () => { warnings += 1 }, error() {} } as unknown as Logger + const manager = new WorkflowManager({ + workspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger, + storageDir, + createClient: () => client, + }) + ;(manager as any).pruneHistory = async () => { throw new Error("prune failed") } + + try { + const started = await manager.start({ + workspaceId: "workspace", + objective: "Finish despite prune failure", + stages: [{ id: "only", title: "Only", instructions: "Complete" }], + }) + let run = started + for (let attempt = 0; attempt < 50 && run.status === "running"; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + run = (await manager.get(started.id))! + } + + assert.equal(run.status, "completed") + assert.equal(warnings, 1) + } finally { + await manager.shutdown() + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) + + it("clears a rejected cached shutdown so shutdown can be retried", async () => { + const storageDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-workflow-shutdown-retry-")) + const manager = new WorkflowManager({ + workspaceManager: { list: () => [] } as unknown as WorkspaceManager, + eventBus: { publish: () => true } as unknown as EventBus, + logger: { warn() {}, error() {} } as unknown as Logger, + storageDir, + }) + await manager.list() + let attempts = 0 + ;(manager as any).performShutdown = async () => { + if (++attempts === 1) throw new Error("shutdown failed") + } + try { + await assert.rejects(manager.shutdown(), /shutdown failed/) + await assert.doesNotReject(manager.shutdown()) + assert.equal(attempts, 2) + } finally { + await fs.rm(storageDir, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workflows/manager.ts b/packages/server/src/workflows/manager.ts new file mode 100644 index 000000000..9b8eb7c2e --- /dev/null +++ b/packages/server/src/workflows/manager.ts @@ -0,0 +1,2400 @@ +import { randomUUID } from "node:crypto" +import { spawnSync } from "node:child_process" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { isDeepStrictEqual } from "node:util" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { + WorkflowDefinitionV1, + WorkflowDefinitionRecord, + WorkflowDefinitionRunCreateRequest, + WorkflowNode, + WorkflowRun, + WorkflowRunStartRequest, + WorkflowRunStep, + WorkflowRunWorktreePolicy, + WorkflowRunWorktreeSelection, + WorkflowSavedDefinitionSnapshot, + WorkspaceDescriptor, +} from "../api-types" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import { createInstanceClient } from "../workspaces/instance-client" +import { createManagedWorktree, isManagedWorktree, isValidWorktreeSlug, listWorktrees, resolveRepoRoot } from "../workspaces/git-worktrees" +import type { WorkspaceManager } from "../workspaces/manager" +import { probePosixProcesses, probeWindowsProcesses } from "../workspaces/process-identity" +import { ensureCodenomadGitExclude } from "../workspaces/worktree-map" +import { WorkflowDefinitionStore } from "./definition-store" +import { validateWorkflowDefinition, WORKFLOW_LIMITS } from "./definition-schema" +import { withFilesystemLock } from "./filesystem-lock" +import { WorkflowCheckpointError, WorkflowInterpreter, WorkflowSuspendedError } from "./interpreter" +import { validateJsonSchemaValue } from "./json-schema" +import { definitionRunFields, holdsWorkflowReservation, isConfirmedRetryCheckpoint, markWorkflowRecoveryRequired, validatePersistedWorkflowRun } from "./run-state" + +const PROMPT_TIMEOUT_MS = 30 * 60 * 1000 +const ABORT_TIMEOUT_MS = 5_000 +const SHUTDOWN_TIMEOUT_MS = 10_000 +const MAX_OUTPUT_CHARS = 16_000 +const WORKFLOW_HISTORY_LIMIT = 100 +const CREATION_CLEANUP_ATTEMPTS = 3 +const EXECUTOR_LEASE_MS = 10_000 +const EXECUTOR_HEARTBEAT_MS = 2_500 +const READ_REFRESH_MS = 250 +const PROCESS_OWNER = currentProcessOwner() + +export class WorkflowRunError extends Error { + constructor(message: string, readonly statusCode: number) { + super(message) + } +} + +interface WorkflowManagerOptions { + workspaceManager: WorkspaceManager + eventBus: EventBus + logger: Logger + storageDir: string + definitionsDir?: string + createClient?: (workspaceId: string) => OpencodeClient | null + promptTimeoutMs?: number +} + +interface ActiveRun { + run: WorkflowRun + client: OpencodeClient + activeSessionId?: string + activeSessionIds: Set + abortingSessions: Map> + sessionCreationsInFlight: number + cancelRequested: boolean + completion?: Promise + abortController: AbortController + releaseBlocked: boolean + pauseCommitted: boolean + leaseFence: number + leaseLost: boolean + leaseDurablyReleased: boolean + heartbeat?: NodeJS.Timeout +} + +interface PendingCancellation { + run: WorkflowRun + active?: ActiveRun + initialDrain?: Promise + finalized?: boolean + unchanged?: boolean +} + +interface DefinitionGraphSnapshot { + root: WorkflowDefinitionV1 + saved: WorkflowSavedDefinitionSnapshot[] +} + +interface RunIndexEntry { + id: string + workspaceId: string + workspaceLineageId: string + workspacePath: string + sourceWorkspaceId?: string + sourceWorkspaceLineageId?: string + sourceWorkspacePath?: string + worktreeDirectory?: string + worktreeSlug?: string + status: WorkflowRun["status"] + createdAt: string + updatedAt: string + ambiguousSessions: boolean +} + +interface RunFileMetadata extends RunIndexEntry { + size: number + mtimeMs: number +} + +export interface WorkflowWorkspaceIdentity { + id?: string + lineageId?: string + path?: string +} + +export interface WorkflowWorktreeIdentity { + slug?: string + path?: string +} + +class WorkflowCancelledError extends Error {} + +export class WorkflowManager { + private readonly activeRuns = new Map() + private readonly activeWorkspaces = new Map() + private readonly reservedLineages = new Map() + private readonly reservedPaths = new Map() + private readonly persistQueues = new Map>() + private readonly transitionQueues = new Map>() + private readonly deferredCreationCleanups = new Map() + private readonly quarantinedWorktrees: Array> = [] + private readonly runIndex = new Map() + private readonly shutdownAbortController = new AbortController() + private readonly createClient: (workspaceId: string) => OpencodeClient | null + private readonly promptTimeoutMs: number + private readonly managerToken = randomUUID() + private readonly processOwner = PROCESS_OWNER + private initialized?: Promise + private readRefresh?: Promise + private lastReadRefreshAt = 0 + private readonly definitionStore: WorkflowDefinitionStore + private admissionQueue = Promise.resolve() + private creationCleanupQueue = Promise.resolve() + private shuttingDown = false + private shutdownPromise?: Promise + private admissionFailure?: string + + constructor(private readonly options: WorkflowManagerOptions) { + this.promptTimeoutMs = options.promptTimeoutMs ?? PROMPT_TIMEOUT_MS + this.createClient = options.createClient + ?? ((workspaceId) => createInstanceClient(options.workspaceManager, workspaceId, { timeoutMs: WORKFLOW_LIMITS.timeoutMs })) + this.definitionStore = new WorkflowDefinitionStore( + options.definitionsDir ?? path.join(options.storageDir, "definitions"), + ) + } + + async start(input: WorkflowRunStartRequest): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + try { + return await this.withAdmission(() => this.startAdmitted(input)) + } finally { + await this.drainCreationCleanups() + } + } + + async startLatest(input: WorkflowDefinitionRunCreateRequest): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + try { + return await this.withAdmission(async () => { + const current = await this.definitionStore.get(input.definitionId) + if (!current) throw new WorkflowRunError("Workflow definition not found", 404) + if (input.definitionRevision !== undefined && current.revision !== input.definitionRevision) { + throw new WorkflowRunError("Workflow definition revision is stale", 409) + } + return this.startAdmitted({ ...input, definitionRevision: current.revision }) + }) + } finally { + await this.drainCreationCleanups() + } + } + + private async startAdmitted(input: WorkflowRunStartRequest): Promise { + this.throwIfShuttingDown() + this.throwIfAdmissionBlocked() + const legacy = "stages" in input + if (!legacy && input.runId) { + const existing = this.activeRuns.get(input.runId)?.run ?? await this.read(input.runId) + if (existing) return this.idempotentStart(existing, input) + } + const definition = legacy ? undefined : await this.definitionStore.get(input.definitionId, input.definitionRevision) + if (!legacy && !definition) throw new WorkflowRunError("Workflow definition not found", 404) + const graph = definition ? await this.snapshotDefinitionGraph(definition) : undefined + this.throwIfShuttingDown() + const selected = legacy + ? this.currentWorkspace(input.workspaceId) + : await this.selectWorktree(input, input.worktree ?? { mode: "current" }) + const workspace = selected.workspace + const creationRequest = "creationRequest" in selected ? selected.creationRequest : undefined + let active: ActiveRun | undefined + let persisted = false + let retained = !creationRequest + try { + await this.assertNoPersistedReservation(workspace) + const client = this.requireReadyClient(workspace.id) + const now = new Date().toISOString() + const run: WorkflowRun = { + id: !legacy && input.runId ? input.runId : randomUUID(), + workspaceId: workspace.id, + workspaceLineageId: workspace.lineageId ?? workspace.id, + workspacePath: workspace.path, + ...(input.initiatorSessionId ? { initiatorSessionId: input.initiatorSessionId } : {}), + objective: input.objective ?? definition!.definition.name, + status: "running", + steps: legacy ? input.stages.map((stage) => ({ ...stage, status: "pending" })) : [], + revision: 0, + ...(definition && !("stages" in input) ? { + ...definitionRunFields(definition, input), + definitionSnapshot: graph!.root, + savedDefinitionSnapshots: graph!.saved, + worktreeSelection: selected.selection, + } : {}), + createdAt: now, + updatedAt: now, + } + this.claimExecutor(run) + active = { + run, client, activeSessionIds: new Set(), abortingSessions: new Map(), sessionCreationsInFlight: 0, cancelRequested: false, + abortController: new AbortController(), releaseBlocked: false, pauseCommitted: false, + leaseFence: run.executorFence!, leaseLost: false, leaseDurablyReleased: false, + } + this.reserve(active) + await this.persist(run) + persisted = true + if (creationRequest) { + this.throwIfShuttingDown() + if (!this.options.workspaceManager.releaseCreationRequest(creationRequest.workspaceId, creationRequest.requestId)) { + throw new WorkflowRunError("Managed worktree workspace ownership could not be retained; the workflow was not started", 500) + } + retained = true + } + this.launch(active, (current) => legacy ? this.executePendingStages(current) : this.executeDefinition(current)) + return run + } catch (error) { + if (active && !retained) { + try { + if (persisted) { + await durableRemove(this.runPath(active.run.id)) + await durableRemove(this.runMetadataPath(active.run.id)) + } + this.runIndex.delete(active.run.id) + this.release(active, true) + } catch (rollbackError) { + active.releaseBlocked = true + this.admissionFailure = `Workflow admission is blocked because startup rollback failed for run ${active.run.id}; remove or repair it and restart CodeNomad` + this.options.logger.error({ err: rollbackError, runId: active.run.id }, "Failed to roll back workflow startup") + } + } else if (active && !persisted) { + this.release(active, true) + } + throw error + } finally { + if (creationRequest && !retained) this.deferCreationCleanup(creationRequest.requestId) + } + } + + private currentWorkspace(workspaceId: string): { workspace: WorkspaceDescriptor; selection?: undefined } { + const workspace = this.options.workspaceManager.get(workspaceId) + if (!workspace) throw new WorkflowRunError("Workspace not found", 404) + if (workspace.status !== "ready") throw new WorkflowRunError("Workspace instance is not ready", 409) + return { workspace } + } + + private idempotentStart(existing: WorkflowRun, input: WorkflowDefinitionRunCreateRequest): WorkflowRun { + const sourceWorkspaceId = existing.worktreeSelection?.sourceWorkspaceId ?? existing.workspaceId + const requestedPolicy = input.worktree ?? { mode: "current" } + if (sourceWorkspaceId !== input.workspaceId || existing.definitionId !== input.definitionId + || (input.definitionRevision !== undefined && existing.definitionRevision !== input.definitionRevision) + || (input.objective !== undefined && existing.objective !== input.objective) + || !isDeepStrictEqual(existing.inputs ?? {}, input.inputs ?? {}) + || !isDeepStrictEqual(existing.worktreeSelection?.policy ?? { mode: "current" }, requestedPolicy)) { + throw new WorkflowRunError("Workflow run ID already belongs to another start request", 409) + } + return existing + } + + private async selectWorktree( + input: WorkflowDefinitionRunCreateRequest, + policy: WorkflowRunWorktreePolicy, + ): Promise<{ + workspace: WorkspaceDescriptor + selection: WorkflowRunWorktreeSelection + creationRequest?: { workspaceId: string; requestId: string } + }> { + const source = this.currentWorkspace(input.workspaceId).workspace + if (policy.mode === "current") return { + workspace: source, + selection: { + policy, + sourceWorkspaceId: source.id, + sourceWorkspaceLineageId: source.lineageId ?? source.id, + sourceWorkspacePath: source.path, + workspaceId: source.id, + directory: source.path, + created: false, + }, + } + if (input.initiatorSessionId) { + throw new WorkflowRunError("initiatorSessionId is unsupported when a workflow selects a different worktree workspace", 400) + } + if (!isValidWorktreeSlug(policy.slug) || policy.slug === "root") { + throw new WorkflowRunError("Invalid workflow worktree slug", 400) + } + + let repository + try { + repository = await resolveRepoRoot(source.path, this.options.logger, { + signal: this.shutdownAbortController.signal, + }) + } catch (error) { + throw new WorkflowRunError(`Workflow worktree selection is unavailable: ${this.errorMessage(error)}`, 409) + } + const { repoRoot, isGitRepo } = repository + if (!isGitRepo) throw new WorkflowRunError("Workflow worktree policy requires a Git repository", 400) + let target: { slug: string; directory: string; branch?: string } + if (policy.mode === "existing") { + const match = (await listWorktrees({ + repoRoot, workspaceFolder: source.path, logger: this.options.logger, + signal: this.shutdownAbortController.signal, + })) + .find((worktree) => worktree.kind === "worktree" && worktree.slug === policy.slug) + if (!match) throw new WorkflowRunError(`Managed worktree ${policy.slug} was not found`, 404) + if (!await isManagedWorktree({ repoRoot, worktree: match })) { + throw new WorkflowRunError(`Worktree ${policy.slug} is not managed by CodeNomad`, 400) + } + target = match + } else { + this.throwIfShuttingDown() + await ensureCodenomadGitExclude(source.path, this.options.logger).catch(() => undefined) + this.throwIfShuttingDown() + try { + target = await createManagedWorktree({ + repoRoot, workspaceFolder: source.path, slug: policy.slug, logger: this.options.logger, + signal: this.shutdownAbortController.signal, + }) + } catch (error) { + throw new WorkflowRunError(`Failed to create managed worktree ${policy.slug}: ${this.errorMessage(error)}`, 409) + } + } + + const requestId = `workflow-${randomUUID()}` + let handedOff = false + try { + this.throwIfShuttingDown() + const createdWorkspace = await this.options.workspaceManager.create(target.directory, `Workflow: ${policy.slug}`, { + requestId, + binaryPath: source.binaryId, + }) + this.throwIfShuttingDown() + handedOff = true + return { + workspace: createdWorkspace.workspace, + creationRequest: { workspaceId: createdWorkspace.workspace.id, requestId }, + selection: { + policy, + sourceWorkspaceId: source.id, + sourceWorkspaceLineageId: source.lineageId ?? source.id, + sourceWorkspacePath: source.path, + workspaceId: createdWorkspace.workspace.id, + directory: createdWorkspace.workspace.path, + slug: target.slug, + ...(target.branch ? { branch: target.branch } : {}), + created: policy.mode === "new", + }, + } + } catch (error) { + if (error instanceof WorkflowRunError) throw error + throw new WorkflowRunError( + `Managed worktree ${policy.slug} was selected, but its OpenCode workspace could not be started: ${this.errorMessage(error)}. The worktree was retained for inspection.`, + 409, + ) + } finally { + if (!handedOff) this.deferCreationCleanup(requestId) + } + } + + private async snapshotDefinitionGraph(root: WorkflowDefinitionRecord): Promise { + const saved = new Map() + const resolved = new Set() + const snapshot = async (record: WorkflowDefinitionRecord, stack: string[], depth: number): Promise => { + if (depth > WORKFLOW_LIMITS.nestingDepth) { + throw new WorkflowRunError(`Saved workflow nesting exceeds maximum depth ${WORKFLOW_LIMITS.nestingDepth}`, 400) + } + if (stack.includes(record.id)) { + throw new WorkflowRunError(`Saved workflow cycle detected: ${[...stack, record.id].join(" -> ")}`, 400) + } + const key = `${record.id}@${record.revision}` + if (resolved.has(key)) return saved.get(key)!.definition + const definition = JSON.parse(JSON.stringify(record.definition)) as WorkflowDefinitionV1 + const nextStack = [...stack, record.id] + const inspect = async (node: WorkflowNode): Promise => { + if (node.type === "workflow") { + const child = await this.definitionStore.get(node.definitionId, node.definitionRevision) + if (!child) { + const revision = node.definitionRevision ? ` revision ${node.definitionRevision}` : "" + throw new WorkflowRunError(`Referenced workflow definition ${node.definitionId}${revision} was not found`, 404) + } + node.definitionRevision = child.revision + const childDefinition = await snapshot(child, nextStack, depth + 1) + const childKey = `${child.id}@${child.revision}` + if (!saved.has(childKey)) saved.set(childKey, { id: child.id, revision: child.revision, definition: childDefinition }) + if (saved.size > WORKFLOW_LIMITS.staticNodes) { + throw new WorkflowRunError(`Saved workflow graph exceeds ${WORKFLOW_LIMITS.staticNodes} definitions`, 400) + } + return + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) await inspect(child) + } + await inspect(definition.root) + resolved.add(key) + return definition + } + const rootDefinition = await snapshot(root, [], 0) + saved.delete(`${root.id}@${root.revision}`) + const snapshots = Array.from(saved.values()) + const graphBytes = Buffer.byteLength(JSON.stringify([rootDefinition, ...snapshots.map((item) => item.definition)]), "utf8") + if (graphBytes > WORKFLOW_LIMITS.sourceBytes) { + throw new WorkflowRunError(`Saved workflow graph exceeds ${WORKFLOW_LIMITS.sourceBytes} bytes`, 400) + } + const byKey = new Map(snapshots.map((item) => [`${item.id}@${item.revision}`, item.definition])) + const limit = rootDefinition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + const add = (left: number, right: number) => Math.min(limit + 1, left + right) + const multiply = (left: number, right: number) => Math.min(limit + 1, left * right) + const expansionMemo = new Map() + const definitionExpansion = (key: string, definition: WorkflowDefinitionV1): number => { + const cached = expansionMemo.get(key) + if (cached !== undefined) return cached + const result = expansion(definition.root) + expansionMemo.set(key, result) + return result + } + const expansion = (node: WorkflowNode): number => { + if (node.type === "workflow") { + const key = `${node.definitionId}@${node.definitionRevision}` + const child = byKey.get(key) + return child ? add(1, definitionExpansion(key, child)) : limit + 1 + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + let total = 1 + for (const child of children) { + total = add(total, expansion(child)) + if (total > limit) return total + } + if (node.type === "foreach") total = add(1, multiply(total - 1, node.maxItems)) + if (node.type === "repeat") total = add(1, multiply(total - 1, node.maxIterations)) + return total + } + const expanded = expansion(rootDefinition.root) + if (expanded > limit) throw new WorkflowRunError(`Saved workflow graph can expand above limit ${limit}`, 400) + return { root: rootDefinition, saved: snapshots } + } + + validateDefinition(source: string | unknown) { return validateWorkflowDefinition(source) } + async createDefinition(source: string | unknown) { + await this.ensureInitialized() + return this.withAdmission(() => this.definitionStore.create(source)) + } + async updateDefinition(id: string, expectedRevision: number, source: string | unknown) { + await this.ensureInitialized() + return this.withAdmission(() => this.definitionStore.update(id, expectedRevision, source)) + } + async deleteDefinition(id: string, expectedRevision: number) { + await this.ensureInitialized() + return this.withAdmission(() => this.definitionStore.delete(id, expectedRevision)) + } + async getDefinition(id: string, revision?: number) { return this.definitionStore.get(id, revision) } + async listDefinitions(): Promise { return this.definitionStore.list() } + + async get(runId: string, workspaceId?: string): Promise { + await this.ensureInitialized() + let active = this.activeRuns.get(runId) + if (!active) { + await this.refreshForRead() + active = this.activeRuns.get(runId) + } + if (!active) { + const run = await this.read(runId) + return workspaceId && run ? this.bindWorkspace(run.id, workspaceId) : run + } + if (["paused", "waiting_for_review", "waiting_for_input", "completed", "failed", "cancelled", "interrupted", "recovery_required"].includes(active.run.status)) { + await active.completion + if (active.releaseBlocked) return workspaceId ? this.bindWorkspace(active.run.id, workspaceId) : active.run + const run = await this.read(runId) ?? active.run + return workspaceId ? this.bindWorkspace(run.id, workspaceId) : run + } + return workspaceId ? this.bindWorkspace(active.run.id, workspaceId) : active.run + } + + async list(workspaceId?: string): Promise { + await this.ensureInitialized() + await this.refreshForRead() + const requested = workspaceId ? this.options.workspaceManager.get(workspaceId) : undefined + const candidates = Array.from(this.runIndex.values()) + .filter((entry) => !workspaceId || this.indexMatchesWorkspace(entry, workspaceId, requested)) + .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) + const matched: WorkflowRun[] = [] + for (const entry of candidates) { + if (matched.length >= WORKFLOW_HISTORY_LIMIT) break + const run = await this.readForListing(`${entry.id}.json`) + if (!run) continue + const bound = workspaceId ? await this.bindWorkspace(run.id, workspaceId) : run + if (bound) matched.push(bound) + } + return matched + } + + /** Uncapped ownership predicate matching execution and source workspaces by ID, canonical lineage, or canonical path. */ + async isWorkspaceWorkflowOwned(identity: WorkflowWorkspaceIdentity): Promise { + await this.ensureInitialized() + return this.withAdmission(() => this.isWorkspaceWorkflowOwnedCurrent(identity)) + } + + /** Holds the same serialized lease as workflow admission through the ownership decision and caller operation. */ + async withWorkspaceOwnershipLease( + identity: WorkflowWorkspaceIdentity, + operation: (owned: boolean) => Promise, + ): Promise { + await this.ensureInitialized() + return this.withAdmission(async () => operation(await this.isWorkspaceWorkflowOwnedCurrent(identity))) + } + + /** Uncapped worktree predicate matching the source canonically and the selected worktree by slug or path. */ + async isWorktreeWorkflowOwned(source: WorkflowWorkspaceIdentity, worktree: WorkflowWorktreeIdentity): Promise { + await this.ensureInitialized() + return this.withAdmission(() => this.isWorktreeWorkflowOwnedCurrent(source, worktree)) + } + + /** Holds workflow admission while a caller checks and deletes a canonical managed worktree. */ + async withWorktreeOwnershipLease( + source: WorkflowWorkspaceIdentity, + worktree: WorkflowWorktreeIdentity, + operation: (owned: boolean) => Promise, + ): Promise { + await this.ensureInitialized() + return this.withAdmission(async () => operation(await this.isWorktreeWorkflowOwnedCurrent(source, worktree))) + } + + async approve(runId: string, expectedStepId: string): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + const candidate = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (candidate?.definitionSnapshot) { + if (candidate.pendingGate?.gate !== "approval") throw new WorkflowRunError("Workflow run is not waiting for approval", 409) + if (candidate.pendingGate.executionNodeId !== expectedStepId) throw new WorkflowRunError("Workflow approval is stale", 409) + return this.answer(runId, candidate.pendingGate.executionNodeId, true) + } + const settlement = await this.waitForControlSettlement(runId, (active) => + active.cancelRequested || active.run.status === "waiting_for_review") + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfShuttingDown() + const current = this.activeRuns.get(runId) + const run = await this.read(runId) + if (!run) return undefined + if (settlement?.active.cancelRequested && ["cancelled", "recovery_required"].includes(run.status)) return run + if (current && current.run.status !== "waiting_for_review") { + throw new WorkflowRunError("Workflow stage is not ready for review", 409) + } + if (settlement && !settlement.settled && current === settlement.active) { + throw new WorkflowRunError("Workflow stage is still settling", 409) + } + if (run.pendingReviewStepId !== expectedStepId) throw new WorkflowRunError("Workflow approval is stale", 409) + const reviewed = run.steps.find((step) => step.id === run.pendingReviewStepId) + if (run.status !== "waiting_for_review" || reviewed?.status !== "completed") { + throw new WorkflowRunError("Workflow run is not waiting for review", 409) + } + + const restoredWorkspace = this.options.workspaceManager.list().find((workspace) => + workspace.lineageId === run.workspaceLineageId && workspace.status === "ready") + if (restoredWorkspace && restoredWorkspace.id !== run.workspaceId) { + await this.bindWorkspaceCurrent(run, restoredWorkspace.id) + } + const workspace = this.options.workspaceManager.get(run.workspaceId) + if (workspace) await this.assertNoPersistedReservation(workspace, run.id) + const client = this.requireReadyClient(run.workspaceId, run.id) + const prior = this.cloneRun(run) + run.status = "running" + delete run.pendingReviewStepId + delete run.error + this.claimExecutor(run) + const active: ActiveRun = { + run, client, activeSessionIds: new Set(), abortingSessions: new Map(), sessionCreationsInFlight: 0, cancelRequested: false, + abortController: new AbortController(), releaseBlocked: false, pauseCommitted: false, + leaseFence: run.executorFence!, leaseLost: false, leaseDurablyReleased: false, + } + this.reserve(active) + try { + await this.persist(run) + } catch (error) { + this.restoreRun(run, prior) + this.release(active) + this.reserveRun(run) + throw error + } + if (active.cancelRequested || this.shuttingDown) { + return this.cancelRunCurrent(runId, run) + } + this.launch(active, (current) => this.executePendingStages(current)) + return run + })) + } + + async answer(runId: string, executionNodeId: string, answer: unknown): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + const settlement = await this.waitForControlSettlement(runId, (active) => + active.cancelRequested || Boolean(active.run.pendingGate + && ["waiting_for_review", "waiting_for_input"].includes(active.run.status))) + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfShuttingDown() + const current = this.activeRuns.get(runId) + const run = await this.read(runId) + if (!run) return undefined + if (settlement?.active.cancelRequested && ["cancelled", "recovery_required"].includes(run.status)) return run + if (current && (!current.run.pendingGate || !["waiting_for_review", "waiting_for_input"].includes(current.run.status))) { + if (current.run.executionNodes?.some((node) => node.id === executionNodeId && node.status === "completed")) { + throw new WorkflowRunError("Workflow gate answer is stale", 409) + } + throw new WorkflowRunError("Workflow run is not waiting for a gate answer", 409) + } + if (settlement && !settlement.settled && current === settlement.active) { + throw new WorkflowRunError("Workflow gate is still settling", 409) + } + const gate = run.pendingGate + if (!run.definitionSnapshot || !gate || !["waiting_for_review", "waiting_for_input"].includes(run.status)) { + throw new WorkflowRunError("Workflow run is not waiting for a gate answer", 409) + } + if (gate.executionNodeId !== executionNodeId) throw new WorkflowRunError("Workflow gate answer is stale", 409) + if (gate.gate === "approval" && answer !== true) { + throw new WorkflowRunError("Approval gates require answer true", 400) + } + if (gate.gate === "input" && gate.inputSchema) { + const issues = validateJsonSchemaValue(answer, gate.inputSchema) + if (issues.length) throw new WorkflowRunError(`Gate answer is invalid: ${issues.join("; ")}`, 400) + } + const serializedAnswer = JSON.stringify(answer) + if (serializedAnswer === undefined || serializedAnswer.length > MAX_OUTPUT_CHARS) throw new WorkflowRunError("Gate answer is too large", 400) + const execution = run.executionNodes?.find((node) => node.id === gate.executionNodeId) + if (!execution || execution.status !== "waiting") throw new WorkflowRunError("Workflow gate state is invalid", 409) + const client = await this.prepareDefinitionWorkspace(run) + this.throwIfShuttingDown() + const prior = this.cloneRun(run) + await this.confirmPersistedSessionAborts(run, client, "Workflow gate answer") + execution.status = "completed" + execution.output = answer + execution.completedAt = new Date().toISOString() + delete run.pendingGate + return this.resumeDefinitionRun(run, client, prior) + })) + } + + async pause(runId: string): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + return this.withAdmission(() => this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (!run) return undefined + this.assertNoLiveForeignLease(run) + if (!run.definitionSnapshot) throw new WorkflowRunError("Legacy workflows cannot be paused", 409) + if (run.status === "paused" || run.status === "pausing") return run + if (run.status !== "running") throw new WorkflowRunError("Workflow run is not running", 409) + const priorStatus = run.status + const priorPauseRequested = run.pauseRequested + run.pauseRequested = true + run.status = "pausing" + try { + await this.persist(run) + const active = this.activeRuns.get(runId) + if (active?.run === run) active.pauseCommitted = true + } catch (error) { + if (run.status === "pausing") run.status = priorStatus + if (run.pauseRequested === true) { + if (priorPauseRequested === undefined) delete run.pauseRequested + else run.pauseRequested = priorPauseRequested + } + throw error + } + return run + })) + } + + async resume(runId: string, confirmRecovery = false, expectedRevision?: number): Promise { + await this.ensureInitialized() + this.throwIfShuttingDown() + const settlement = await this.waitForControlSettlement(runId, (active) => + active.cancelRequested || ["paused", "interrupted", "recovery_required"].includes(active.run.status)) + return this.withAdmission(() => this.withRunTransition(runId, async () => { + this.throwIfAdmissionBlocked() + const active = this.activeRuns.get(runId) + const run = await this.read(runId) + if (!run) return undefined + if (settlement?.active.cancelRequested && ["cancelled", "recovery_required"].includes(run.status)) return run + if (active && !["paused", "interrupted", "recovery_required"].includes(active.run.status)) { + throw new WorkflowRunError("Workflow run cannot be resumed", 409) + } + if (settlement && !settlement.settled && active === settlement.active) { + throw new WorkflowRunError("Workflow run is still settling", 409) + } + if (!run.definitionSnapshot) throw new WorkflowRunError("Legacy workflows cannot be resumed", 409) + if (run.status === "recovery_required" && !confirmRecovery) { + throw new WorkflowRunError("Recovery confirmation is required before repeating an ambiguous side effect", 409) + } + if (confirmRecovery && (run.status !== "recovery_required" || expectedRevision !== run.revision)) { + throw new WorkflowRunError("Workflow recovery confirmation is stale", 409) + } + if (!["paused", "interrupted", "recovery_required"].includes(run.status)) { + throw new WorkflowRunError("Workflow run cannot be resumed", 409) + } + const client = await this.prepareDefinitionWorkspace(run) + this.throwIfShuttingDown() + const prior = this.cloneRun(run) + if (run.status === "recovery_required") { + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size === 0 && this.hasUnconfirmedAdmittedAction(run)) { + const message = "Workflow recovery has no persisted session IDs and cannot positively confirm termination; the interrupted action will not be repeated" + await this.persistMutation(run, () => markWorkflowRecoveryRequired(run, message)) + throw new WorkflowRunError(message, 409) + } + await this.confirmPersistedSessionAborts(run, client, "Workflow recovery") + for (const node of run.executionNodes ?? []) if (node.status === "interrupted") { + node.status = "pending" + node.attempt = 0 + delete node.error + delete node.completedAt + } + } else { + await this.confirmPersistedSessionAborts(run, client, "Workflow resume") + } + if (active) { + active.activeSessionIds.clear() + active.activeSessionId = undefined + active.releaseBlocked = false + if (this.activeRuns.get(run.id) === active) this.activeRuns.delete(run.id) + } + return this.resumeDefinitionRun(run, client, prior) + })) + } + + async cancel(runId: string): Promise { + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + const active = this.activeRuns.get(runId) + if (active) { + active.cancelRequested = true + active.releaseBlocked = true + } + await this.ensureInitialized() + return this.cancelRun(runId) + } + + /** Atomically verifies plugin workspace ownership and cancels without joining/rebinding the run via get(). */ + async cancelOwned(runId: string, workspaceId: string): Promise { + await this.ensureInitialized() + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + const pending = await this.withAdmission(() => this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + if (!run) return undefined + const workspace = this.options.workspaceManager.get(workspaceId) + const executionMatch = run.workspaceId === workspaceId || Boolean(workspace?.status === "ready" + && this.matchesCanonicalWorkspace(workspace, run.workspaceLineageId, run.workspacePath)) + const selection = run.worktreeSelection + const sourceMatch = selection?.sourceWorkspaceId === workspaceId || Boolean(selection && workspace?.status === "ready" + && this.matchesCanonicalWorkspace(workspace, selection.sourceWorkspaceLineageId, selection.sourceWorkspacePath)) + if (!executionMatch && !sourceMatch) return undefined + if (executionMatch && run.workspaceId !== workspaceId) { + if (!await this.bindWorkspaceCurrent(run, workspaceId)) return undefined + } + return this.fenceCancellation(runId, run) + })) + return pending && this.finalizeCancellation(pending) + } + + async shutdown(): Promise { + if (!this.shutdownPromise) { + this.shuttingDown = true + this.shutdownAbortController.abort(new WorkflowCancelledError()) + this.shutdownPromise = this.performShutdown() + } + const pending = this.shutdownPromise + try { + await this.withTimeout(pending, SHUTDOWN_TIMEOUT_MS, "Workflow shutdown timed out") + } catch (error) { + if (this.shutdownPromise === pending) this.shutdownPromise = undefined + throw error + } + } + + private async cancelRun(runId: string): Promise { + const pending = await this.withAdmission(() => this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + return run ? this.fenceCancellation(runId, run) : undefined + })) + return pending && this.finalizeCancellation(pending) + } + + private async cancelRunCurrent(runId: string, run: WorkflowRun): Promise { + return this.finalizeCancellation(await this.fenceCancellation(runId, run)) + } + + private async fenceCancellation(runId: string, run: WorkflowRun): Promise { + const active = this.activeRuns.get(runId) + this.assertNoLiveForeignLease(run) + if (!["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"].includes(run.status)) { + return { run, unchanged: true } + } + + if (!active && this.persistedAmbiguousSessionIds(run).size === 0 && !this.hasUnconfirmedAdmittedAction(run)) { + await this.persistMutation(run, () => { + this.markCancelled(run) + this.clearExpiredExecutorLease(run) + this.releaseExecutorLease(run) + }) + return { run, finalized: true } + } + + const priorCancelRequested = active?.cancelRequested + const priorReleaseBlocked = active?.releaseBlocked + if (active) { + active.cancelRequested = true + active.releaseBlocked = true + } + try { + await this.persistMutation(run, () => { + run.status = "recovery_required" + }) + } catch (error) { + if (active) { + active.cancelRequested = priorCancelRequested! + active.releaseBlocked = priorReleaseBlocked! + } + throw error + } + if (!active) return { run } + this.requestCancellation(active) + return { run, active, initialDrain: this.drainSessions(active) } + } + + private async finalizeCancellation(pending: PendingCancellation): Promise { + const { run, active } = pending + if (pending.unchanged) return run + let terminationConfirmed = true + let completionSettled = true + if (active) { + terminationConfirmed = await pending.initialDrain! + completionSettled = !active.completion || await this.withTimeout( + active.completion.then(() => true), + Math.max(1, Math.min(ABORT_TIMEOUT_MS, this.promptTimeoutMs)), + "Workflow cancellation settlement timed out", + ).catch(() => false) + terminationConfirmed = terminationConfirmed && await this.drainSessions(active) + if (!completionSettled && (active.sessionCreationsInFlight > 0 + || (this.persistedAmbiguousSessionIds(run).size === 0 && this.hasUnconfirmedAdmittedAction(run)))) { + terminationConfirmed = false + } + if (terminationConfirmed && completionSettled) { + this.clearPersistedSessions(run, this.persistedAmbiguousSessionIds(run)) + } + } + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size > 0) { + const client = active?.client ?? await this.prepareDefinitionWorkspace(run) + terminationConfirmed = terminationConfirmed && (await Promise.all(Array.from(sessionIds).map((sessionId) => + this.abortSessionRequest(client, run.id, sessionId)))).every(Boolean) + } else if (run.status === "recovery_required" && !active) { + terminationConfirmed = !this.hasUnconfirmedAdmittedAction(run) + } + const message = sessionIds.size === 0 + ? "Workflow cancellation has no persisted session IDs and cannot positively confirm termination" + : "Workflow cancellation could not confirm every session abort" + try { + if (!pending.finalized) await this.persistMutation(run, () => { + if (terminationConfirmed) { + this.clearPersistedSessions(run, sessionIds) + this.markCancelled(run) + this.clearExpiredExecutorLease(run) + this.releaseExecutorLease(run) + } else { + markWorkflowRecoveryRequired(run, message) + this.clearExpiredExecutorLease(run) + this.releaseExecutorLease(run) + } + }) + if (terminationConfirmed) await durableRemove(this.recoveryMarkerPath(run.id)).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to clear resolved workflow recovery marker") + }) + } catch (error) { + if (active) active.releaseBlocked = true + this.reserveRun(run) + throw error + } + if (active) { + active.releaseBlocked = !terminationConfirmed + if (terminationConfirmed) this.release(active) + } + if (!holdsWorkflowReservation(run) && this.activeWorkspaces.get(run.workspaceId) === run.id) { + this.activeWorkspaces.delete(run.workspaceId) + } + if (!holdsWorkflowReservation(run) && this.reservedLineages.get(run.workspaceLineageId) === run.id) { + this.reservedLineages.delete(run.workspaceLineageId) + } + if (!holdsWorkflowReservation(run) && this.reservedPaths.get(this.pathKey(run.workspacePath)) === run.id) { + this.reservedPaths.delete(this.pathKey(run.workspacePath)) + } + return run + } + + private async performShutdown(): Promise { + await this.ensureInitialized() + await this.admissionQueue.catch(() => undefined) + await this.drainCreationCleanups(true) + const active = Array.from(this.activeRuns.values()) + for (const run of active) this.requestCancellation(run) + await Promise.all(active.map(async (item) => { + try { + await this.cancelRun(item.run.id) + } catch (error) { + if (!(error instanceof WorkflowRunError) || error.statusCode !== 409) throw error + this.release(item, true) + } + })) + await Promise.all(active.map(({ completion }) => completion)) + await Promise.all(Array.from(this.transitionQueues.values())) + await Promise.all(Array.from(this.persistQueues.values())) + } + + private async resumeDefinitionRun(run: WorkflowRun, client: OpencodeClient, prior = this.cloneRun(run)): Promise { + this.throwIfShuttingDown() + run.status = "running" + run.pauseRequested = false + delete run.error + this.claimExecutor(run) + const active: ActiveRun = { + run, + client, + activeSessionIds: new Set(), + abortingSessions: new Map(), + sessionCreationsInFlight: 0, + cancelRequested: false, + abortController: new AbortController(), + releaseBlocked: false, + pauseCommitted: false, + leaseFence: run.executorFence!, + leaseLost: false, + leaseDurablyReleased: false, + } + this.reserve(active) + try { + await this.persist(run) + await durableRemove(this.recoveryMarkerPath(run.id)).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to clear resolved workflow recovery marker") + }) + } catch (error) { + this.restoreRun(run, prior) + active.releaseBlocked = false + this.release(active) + this.reserveRun(run) + throw error + } + this.launch(active, (current) => this.executeDefinition(current)) + return run + } + + private async prepareDefinitionWorkspace(run: WorkflowRun): Promise { + this.throwIfShuttingDown() + const lineageWorkspace = this.options.workspaceManager.list().find((workspace) => + workspace.status === "ready" && this.matchesCanonicalWorkspace(workspace, run.workspaceLineageId, run.workspacePath)) + if (lineageWorkspace && lineageWorkspace.id !== run.workspaceId) { + this.throwIfShuttingDown() + await this.bindWorkspaceCurrent(run, lineageWorkspace.id) + } + this.throwIfShuttingDown() + const workspace = this.options.workspaceManager.get(run.workspaceId) + if (workspace) await this.assertNoPersistedReservation(workspace, run.id) + return this.requireReadyClient(run.workspaceId, run.id) + } + + private async withTimeout(operation: Promise, timeoutMs: number, message: string): Promise { + let timeout: NodeJS.Timeout | undefined + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs) + }), + ]) + } finally { + if (timeout) clearTimeout(timeout) + } + } + + private requireReadyClient(workspaceId: string, runId?: string): OpencodeClient { + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + this.throwIfAdmissionBlocked() + const workspace = this.options.workspaceManager.get(workspaceId) + if (!workspace) throw new WorkflowRunError("Workspace not found", 404) + if (workspace.status !== "ready") throw new WorkflowRunError("Workspace instance is not ready", 409) + const activeRunId = this.activeWorkspaces.get(workspaceId) + if (activeRunId && activeRunId !== runId) { + throw new WorkflowRunError("A workflow is already running in this workspace", 409) + } + const lineageId = workspace.lineageId ?? workspace.id + const lineageRunId = this.reservedLineages.get(lineageId) + if (lineageRunId && lineageRunId !== runId) { + throw new WorkflowRunError("A workflow is already running for this workspace lineage", 409) + } + const pathRunId = this.reservedPaths.get(this.pathKey(workspace.path)) + if (pathRunId && pathRunId !== runId) { + throw new WorkflowRunError("A workflow is already running for this workspace path", 409) + } + const client = this.createClient(workspaceId) + if (!client) throw new WorkflowRunError("Workspace instance is not ready", 409) + return client + } + + private async waitForControlSettlement( + runId: string, + shouldWait: (active: ActiveRun) => boolean, + ): Promise<{ active: ActiveRun; settled: boolean } | undefined> { + const active = this.activeRuns.get(runId) + if (!active) return undefined + if (!active.completion || !shouldWait(active)) return { active, settled: true } + const settled = await this.withTimeout( + active.completion.then(() => true), + Math.max(1, Math.min(ABORT_TIMEOUT_MS, this.promptTimeoutMs)), + "Workflow control settlement timed out", + ).catch(() => false) + return { active, settled } + } + + private async assertNoPersistedReservation(workspace: WorkspaceDescriptor, runId?: string): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.options.storageDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + for (const entry of entries) { + if (!entry.endsWith(".json") || entry === `${runId}.json`) continue + const persistedId = entry.slice(0, -5) + let run: WorkflowRun + try { + run = JSON.parse(await fs.readFile(path.join(this.options.storageDir, entry), "utf8")) as WorkflowRun + validatePersistedWorkflowRun(run, persistedId) + } catch (error) { + this.options.logger.warn({ err: error, file: entry }, "Skipping corrupt workflow run during admission") + continue + } + if (!holdsWorkflowReservation(run)) continue + if (run.workspaceLineageId === (workspace.lineageId ?? workspace.id)) { + throw new WorkflowRunError("A workflow is already running for this workspace lineage", 409) + } + if (this.samePath(run.workspacePath, workspace.path)) { + throw new WorkflowRunError("A workflow is already running for this workspace path", 409) + } + } + } + + private throwIfShuttingDown(): void { + if (this.shuttingDown) throw new WorkflowRunError("CodeNomad is shutting down", 503) + } + + private throwIfAdmissionBlocked(): void { + if (this.admissionFailure) throw new WorkflowRunError(this.admissionFailure, 503) + } + + private requestCancellation(active: ActiveRun): void { + active.releaseBlocked = true + active.cancelRequested = true + active.abortController.abort(new WorkflowCancelledError("Workflow run cancelled")) + } + + private async drainSessions(active: ActiveRun): Promise { + const sessionIds = new Set(active.activeSessionIds) + if (active.activeSessionId) sessionIds.add(active.activeSessionId) + const results = await Promise.all(Array.from(sessionIds).map((sessionId) => this.abortActiveSession(active, sessionId))) + return results.every(Boolean) && active.activeSessionIds.size === 0 && active.activeSessionId === undefined + } + + private reserve(active: ActiveRun) { + this.activeRuns.set(active.run.id, active) + // ponytail: one run per workspace; add worktree-aware concurrency only when parallel workflows are needed. + this.reserveRun(active.run) + } + + private reserveRun(run: WorkflowRun) { + this.activeWorkspaces.set(run.workspaceId, run.id) + this.reservedLineages.set(run.workspaceLineageId, run.id) + this.reservedPaths.set(this.pathKey(run.workspacePath), run.id) + } + + private release(active: ActiveRun, force = false) { + if (this.activeRuns.get(active.run.id) !== active) return + if (!force && active.run.definitionSnapshot && this.persistedAmbiguousSessionIds(active.run).size > 0) active.releaseBlocked = true + if (active.releaseBlocked && !force) { + this.options.logger.error({ runId: active.run.id }, "Retaining workflow reservation after unconfirmed session abort") + return + } + if (active.heartbeat) clearTimeout(active.heartbeat) + this.activeRuns.delete(active.run.id) + if ((force || !holdsWorkflowReservation(active.run)) && this.activeWorkspaces.get(active.run.workspaceId) === active.run.id) { + this.activeWorkspaces.delete(active.run.workspaceId) + } + if ((force || !holdsWorkflowReservation(active.run)) && this.reservedLineages.get(active.run.workspaceLineageId) === active.run.id) { + this.reservedLineages.delete(active.run.workspaceLineageId) + } + const pathKey = this.pathKey(active.run.workspacePath) + if ((force || !holdsWorkflowReservation(active.run)) && this.reservedPaths.get(pathKey) === active.run.id) { + this.reservedPaths.delete(pathKey) + } + } + + private withAdmission(operation: () => Promise): Promise { + const admitted = this.admissionQueue.catch(() => undefined) + .then(() => withFilesystemLock(this.admissionLockPath(), async () => { + await this.refreshRunIndex() + await this.recoverInterruptedRuns() + return operation() + })) + this.admissionQueue = admitted.then(() => undefined, () => undefined) + return admitted + } + + private refreshForRead(): Promise { + if (this.readRefresh) return this.readRefresh + if (Date.now() - this.lastReadRefreshAt < READ_REFRESH_MS) return Promise.resolve() + const refresh = this.withAdmission(async () => undefined) + .then(() => { this.lastReadRefreshAt = Date.now() }) + .finally(() => { if (this.readRefresh === refresh) this.readRefresh = undefined }) + this.readRefresh = refresh + return refresh + } + + private admissionLockPath(): string { + return path.join(this.options.storageDir, ".admission.lock") + } + + private async ensureInitialized(): Promise { + if (!this.initialized) { + const pending = withFilesystemLock(this.admissionLockPath(), () => this.recoverInterruptedRuns()) + const attempt = pending.catch((error) => { + if (this.initialized === attempt) this.initialized = undefined + throw error + }) + this.initialized = attempt + } + return this.initialized + } + + private claimExecutor(run: WorkflowRun): void { + this.assertNoLiveForeignLease(run) + const fence = (run.executorFence ?? 0) + 1 + const heartbeatAt = new Date().toISOString() + run.executorFence = fence + run.executorLease = { + ownerToken: this.managerToken, fence, heartbeatAt, expiresAt: this.executorLeaseExpiry(heartbeatAt), + hostname: this.processOwner.hostname, pid: this.processOwner.pid, + ...(this.processOwner.processStart ? { processStart: this.processOwner.processStart } : {}), + ...(this.processOwner.bootId ? { bootId: this.processOwner.bootId } : {}), + } + } + + private releaseExecutorLease(run: WorkflowRun): void { + if (run.executorLease?.ownerToken !== this.managerToken) return + delete run.executorLease + const active = this.activeRuns.get(run.id) + if (active?.heartbeat) { + clearTimeout(active.heartbeat) + active.heartbeat = undefined + } + } + + private clearExpiredExecutorLease(run: WorkflowRun): void { + if (run.executorLease && !this.isLeaseLive(run)) delete run.executorLease + } + + private assertNoLiveForeignLease(run: WorkflowRun): void { + const lease = run.executorLease + if (lease && lease.ownerToken !== this.managerToken && this.isLeaseLive(run)) { + throw new WorkflowRunError("Workflow run is executing on another CodeNomad host", 409) + } + } + + private isLeaseLive(run: WorkflowRun): boolean { + const lease = run.executorLease + if (!lease) return false + if (!lease.hostname || !lease.pid) return Date.parse(lease.expiresAt) > Date.now() + if (lease.hostname !== this.processOwner.hostname) return true + if (lease.pid === this.processOwner.pid) { + return !lease.processStart || !this.processOwner.processStart + || (lease.processStart === this.processOwner.processStart && (!lease.bootId || lease.bootId === this.processOwner.bootId)) + } + const liveness = executorProcessLiveness(lease.pid, lease.processStart, lease.bootId) + return liveness === "alive" || liveness === "unknown" + } + + private executorLeaseExpiry(heartbeatAt = new Date().toISOString()): string { + return new Date(Date.parse(heartbeatAt) + EXECUTOR_LEASE_MS).toISOString() + } + + private deferCreationCleanup(requestId: string): void { + if (!this.deferredCreationCleanups.has(requestId)) this.deferredCreationCleanups.set(requestId, 0) + } + + private async drainCreationCleanups(retryUntilExhausted = false): Promise { + const pending = this.creationCleanupQueue.catch(() => undefined).then(async () => { + const passes = retryUntilExhausted ? CREATION_CLEANUP_ATTEMPTS : 1 + for (let pass = 0; pass < passes; pass += 1) { + const cleanups = Array.from(this.deferredCreationCleanups) + if (cleanups.length === 0) return + for (const [requestId, attempts] of cleanups) { + try { + await this.options.workspaceManager.cancelCreationRequest(requestId) + this.deferredCreationCleanups.delete(requestId) + } catch (error) { + const nextAttempts = attempts + 1 + this.deferredCreationCleanups.set(requestId, nextAttempts) + this.options.logger.warn( + { err: error, requestId, attempt: nextAttempts }, + nextAttempts >= CREATION_CLEANUP_ATTEMPTS + ? "Workflow workspace creation cleanup remains deferred after repeated failures" + : "Workflow workspace creation cleanup will be retried", + ) + } + } + if (!retryUntilExhausted) return + } + if (this.deferredCreationCleanups.size > 0) { + throw new Error(`Workflow workspace creation cleanup remains pending for ${this.deferredCreationCleanups.size} request(s)`) + } + }) + this.creationCleanupQueue = pending.then(() => undefined, () => undefined) + await pending + } + + private async withRunTransition(runId: string, transition: () => Promise): Promise { + const previous = this.transitionQueues.get(runId) ?? Promise.resolve() + const queued = previous.catch(() => undefined).then(transition) + const marker = queued.then(() => undefined, () => undefined) + this.transitionQueues.set(runId, marker) + try { + return await queued + } finally { + if (this.transitionQueues.get(runId) === marker) this.transitionQueues.delete(runId) + } + } + + private launch(active: ActiveRun, execute: (active: ActiveRun) => Promise) { + this.scheduleExecutorHeartbeat(active) + active.completion = execute(active) + .catch((error) => this.handleExecutionError(active, error)) + .catch(async (error) => { + const message = `Workflow recovery is required because terminal state could not be persisted: ${this.errorMessage(error)}` + const blockedMessage = `Workflow admission is blocked because recovery state could not be recorded for run ${active.run.id}; repair storage and restart CodeNomad` + markWorkflowRecoveryRequired(active.run, message) + active.releaseBlocked = true + this.reserveRun(active.run) + if (!this.admissionFailure) this.admissionFailure = blockedMessage + await this.abandonExecutorLease(active).then(() => this.writeRecoveryMarker(active.run, message)).then(() => { + if (this.admissionFailure === blockedMessage) this.admissionFailure = undefined + }).catch((markerError) => { + this.options.logger.error({ err: markerError, runId: active.run.id }, "Failed to write workflow recovery marker") + }) + this.options.logger.error({ err: error, runId: active.run.id }, "Failed to persist workflow failure") + }) + .finally(() => this.release(active)) + } + + private scheduleExecutorHeartbeat(active: ActiveRun): void { + active.heartbeat = setTimeout(() => { + void this.renewExecutorLease(active).then(() => { + if (active.run.executorLease?.ownerToken === this.managerToken + && active.run.executorLease.fence === active.leaseFence) this.scheduleExecutorHeartbeat(active) + }).catch((error) => { + if (active.run.executorLease?.ownerToken !== this.managerToken + || active.run.executorLease.fence !== active.leaseFence) return + active.leaseLost = true + active.releaseBlocked = true + active.cancelRequested = true + active.abortController.abort(new WorkflowCancelledError("Workflow executor lease was lost")) + void this.drainSessions(active) + this.options.logger.error({ err: error, runId: active.run.id }, "Workflow executor lease heartbeat failed") + }) + }, EXECUTOR_HEARTBEAT_MS) + active.heartbeat.unref() + } + + private async executePendingStages(active: ActiveRun): Promise { + const { run, client } = active + if (!run.rootSessionId) { + const root = await this.createTrackedSession(active, client.session.create({ + ...(run.initiatorSessionId ? { parentID: run.initiatorSessionId } : {}), + title: `Workflow: ${run.objective.slice(0, 80)}`, + metadata: this.sessionMetadata(run.id, "workflow"), + }, { signal: this.operationSignal(active) }), "create workflow session") + this.throwIfCancelled(active) + run.rootSessionId = root.id + await this.persist(run) + active.activeSessionIds.delete(root.id) + } + + while (true) { + const index = run.steps.findIndex((step) => step.status === "pending") + if (index < 0) break + const step = run.steps[index]! + const previous = index > 0 ? run.steps[index - 1]?.output : undefined + await this.runStep(active, step, this.buildStagePrompt(run, step, previous)) + this.throwIfCancelled(active) + if (step.requiresApproval) { + run.status = "waiting_for_review" + run.pendingReviewStepId = step.id + delete run.activeStepId + this.releaseExecutorLease(run) + await this.persist(run) + return + } + } + + run.status = "completed" + delete run.activeStepId + delete run.pendingReviewStepId + this.releaseExecutorLease(run) + await this.persist(run) + } + + private cloneRun(run: WorkflowRun): WorkflowRun { + return JSON.parse(JSON.stringify(run)) as WorkflowRun + } + + private restoreRun(run: WorkflowRun, snapshot: WorkflowRun): void { + const currentNodes = run.executionNodes + const restored = this.cloneRun(snapshot) + for (const key of Object.keys(run) as Array) delete run[key] + Object.assign(run, restored) + if (!currentNodes || !restored.executionNodes) return + const byId = new Map(currentNodes.map((node) => [node.id, node])) + const nodes = restored.executionNodes.map((snapshotNode) => { + const current = byId.get(snapshotNode.id) + if (!current) return snapshotNode + for (const key of Object.keys(current) as Array) delete current[key] + Object.assign(current, snapshotNode) + return current + }) + currentNodes.splice(0, currentNodes.length, ...nodes) + run.executionNodes = currentNodes + } + + private async persistMutation(run: WorkflowRun, mutate: () => void): Promise { + const prior = this.cloneRun(run) + mutate() + try { + await this.persist(run) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + + private async executeDefinition(active: ActiveRun): Promise { + const interpreter = new WorkflowInterpreter({ + run: active.run, + client: active.client, + persist: () => this.persist(active.run), + signal: (timeoutMs) => AbortSignal.any([active.abortController.signal, AbortSignal.timeout(timeoutMs)]), + sessionStarted: (sessionId) => { + active.activeSessionIds.add(sessionId) + return !active.cancelRequested + }, + sessionFinished: (sessionId) => active.activeSessionIds.delete(sessionId), + sessionCreationStarted: () => { active.sessionCreationsInFlight += 1 }, + sessionCreationFinished: () => { active.sessionCreationsInFlight -= 1 }, + abortSession: async (sessionId) => { + const confirmed = await this.abortActiveSession(active, sessionId) + if (!confirmed) active.releaseBlocked = true + return confirmed + }, + isCancelled: () => active.cancelRequested, + revalidateFence: () => this.revalidateExecutorFence(active), + isPauseCommitted: () => active.pauseCommitted, + }) + try { + await interpreter.execute() + } catch (error) { + if (error instanceof WorkflowSuspendedError) { + if (active.pauseCommitted && active.run.pauseRequested) { + const priorStatus = active.run.status + active.run.status = "paused" + this.releaseExecutorLease(active.run) + try { + await this.persist(active.run) + } catch (persistError) { + if (active.run.status === "paused") active.run.status = priorStatus + throw persistError + } + } else { + this.releaseExecutorLease(active.run) + await this.persist(active.run) + } + return + } + throw error + } + this.throwIfCancelled(active) + active.run.status = "completed" + active.run.pauseRequested = false + delete active.run.pendingGate + this.releaseExecutorLease(active.run) + try { + await this.persist(active.run) + } catch (error) { + throw new WorkflowCheckpointError(`Workflow completed, but its terminal checkpoint could not be persisted: ${this.errorMessage(error)}`) + } + } + + private buildStagePrompt(run: WorkflowRun, step: WorkflowRunStep, previous: unknown): string { + return [ + `Workflow stage: ${step.title}`, + "", + `Objective:\n${run.objective}`, + "", + `Stage instructions:\n${step.instructions}`, + ...(previous === undefined ? [] : ["", `Previous stage handoff:\n${JSON.stringify(previous, null, 2)}`]), + ].join("\n") + } + + private async runStep( + active: ActiveRun, + step: WorkflowRunStep, + prompt: string, + ): Promise { + const { run, client } = active + this.throwIfCancelled(active) + step.status = "running" + step.startedAt = new Date().toISOString() + run.activeStepId = step.id + await this.persist(run) + this.throwIfCancelled(active) + + const session = await this.createTrackedSession(active, client.session.create({ + parentID: run.rootSessionId, + title: `${step.title}: ${run.objective.slice(0, 60)}`, + ...(step.agent ? { agent: step.agent } : {}), + metadata: this.sessionMetadata(run.id, step.id), + }, { signal: this.operationSignal(active) }), `create ${step.title} session`) + step.sessionId = session.id + active.activeSessionId = session.id + active.activeSessionIds.delete(session.id) + this.throwIfCancelled(active) + await this.persist(run) + this.throwIfCancelled(active) + await this.revalidateExecutorFence(active) + this.throwIfCancelled(active) + + const response = await this.requireData(client.session.prompt({ + sessionID: session.id, + ...(step.agent ? { agent: step.agent } : {}), + ...(step.model ? { model: step.model } : {}), + parts: [{ type: "text", text: prompt }], + }, { signal: this.operationSignal(active) }), `run ${step.title} session`) + active.activeSessionId = undefined + this.throwIfCancelled(active) + if (response.info.error) throw new Error(this.errorMessage(response.info.error)) + + const output = response.info.structured ?? response.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") + const bounded = this.boundOutput(output) + step.output = bounded.output + step.outputTruncated = bounded.truncated || undefined + step.status = "completed" + step.completedAt = new Date().toISOString() + active.activeSessionId = undefined + await this.persist(run) + return bounded.output + } + + private sessionMetadata(runId: string, role: string) { + return { codenomad: { version: 1, workflow: { runId, role } } } + } + + private operationSignal(active: ActiveRun): AbortSignal { + return AbortSignal.any([active.abortController.signal, AbortSignal.timeout(this.promptTimeoutMs)]) + } + + private boundOutput(output: unknown): { output: unknown; truncated: boolean } { + if (typeof output === "string") { + return output.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: output.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + const serialized = JSON.stringify(output) + return serialized.length <= MAX_OUTPUT_CHARS + ? { output, truncated: false } + : { output: serialized.slice(0, MAX_OUTPUT_CHARS), truncated: true } + } + + private throwIfCancelled(active: ActiveRun) { + if (active.cancelRequested) throw new WorkflowCancelledError("Workflow run cancelled") + } + + private async abortSessionRequest(client: OpencodeClient, runId: string, sessionId: string): Promise { + try { + const response = await client.session.abort( + { sessionID: sessionId }, + { signal: AbortSignal.timeout(ABORT_TIMEOUT_MS) }, + ) + if (response.data === true && response.error === undefined) return true + this.options.logger.warn({ err: response.error, runId }, "Workflow session abort was not confirmed") + return false + } catch (error) { + this.options.logger.warn({ err: error, runId }, "Failed to abort workflow session") + return false + } + } + + private async createTrackedSession( + active: ActiveRun, + request: Promise<{ data?: T; error?: unknown }>, + action: string, + ): Promise { + active.sessionCreationsInFlight += 1 + try { + const session = await this.requireData(request, action) + active.activeSessionIds.add(session.id) + return session + } finally { + active.sessionCreationsInFlight -= 1 + } + } + + private async abortActiveSession(active: ActiveRun, sessionId: string): Promise { + const existing = active.abortingSessions.get(sessionId) + if (existing) return existing + if (!active.activeSessionIds.has(sessionId) && active.activeSessionId !== sessionId) return true + const pending = this.abortSessionRequest(active.client, active.run.id, sessionId).then((confirmed) => { + if (confirmed) { + active.activeSessionIds.delete(sessionId) + if (active.activeSessionId === sessionId) active.activeSessionId = undefined + } + return confirmed + }).finally(() => active.abortingSessions.delete(sessionId)) + active.abortingSessions.set(sessionId, pending) + return pending + } + + private async handleExecutionError(active: ActiveRun, error: unknown): Promise { + const { run } = active + if (error instanceof WorkflowCheckpointError) { + markWorkflowRecoveryRequired(run, this.errorMessage(error)) + this.releaseExecutorLease(run) + await this.persist(run) + return + } + if (active.activeSessionId) { + const sessionId = active.activeSessionId + if (!await this.abortActiveSession(active, sessionId)) active.releaseBlocked = true + } + if (run.definitionSnapshot && !await this.drainSessions(active)) active.releaseBlocked = true + if (active.cancelRequested) return + if (active.releaseBlocked) { + const message = `Workflow session abort was not confirmed: ${this.errorMessage(error)}` + markWorkflowRecoveryRequired(run, message) + this.releaseExecutorLease(run) + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = message + step.completedAt = new Date().toISOString() + } + await this.persist(run) + return + } + if (active.cancelRequested || error instanceof WorkflowCancelledError) { + this.markCancelled(run) + this.releaseExecutorLease(run) + } else { + const message = this.errorMessage(error) + run.status = "failed" + run.error = message + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = message + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting") continue + node.status = "failed" + node.error = message + node.completedAt = new Date().toISOString() + } + delete run.pendingGate + delete run.activeStepId + this.releaseExecutorLease(run) + this.options.logger.error({ err: error, runId: run.id }, "Workflow run failed") + } + await this.persist(run) + } + + private markCancelled(run: WorkflowRun) { + run.status = "cancelled" + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "cancelled" + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting") continue + node.status = "cancelled" + node.completedAt = new Date().toISOString() + } + run.pauseRequested = false + delete run.pendingGate + delete run.activeStepId + delete run.pendingReviewStepId + } + + private async requireData(request: Promise<{ data?: T; error?: unknown }>, action: string): Promise { + const response = await request + if (response.data !== undefined) return response.data + throw new Error(`${action} failed: ${this.errorMessage(response.error)}`) + } + + private errorMessage(error: unknown): string { + if (error instanceof Error) return error.message + if (typeof error === "string") return error + try { + return JSON.stringify(error) || "Unknown error" + } catch { + return "Unknown error" + } + } + + private runPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.json`) + } + + private runMetadataPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.meta`) + } + + private recoveryMarkerPath(runId: string) { + return path.join(this.options.storageDir, `${runId}.recovery`) + } + + private runLockPath(runId: string) { + return path.join(this.options.storageDir, ".run-locks", `${runId}.lock`) + } + + private async writeRecoveryMarker(run: WorkflowRun, message: string): Promise { + await ensureDurableDirectory(this.options.storageDir) + await durableAtomicWrite( + this.recoveryMarkerPath(run.id), + JSON.stringify({ runId: run.id, revision: run.revision ?? 0, message, createdAt: new Date().toISOString() }), + ) + } + + private async readRecoveryMarker(runId: string): Promise<{ revision?: number } | undefined> { + try { + const parsed = JSON.parse(await fs.readFile(this.recoveryMarkerPath(runId), "utf8")) as { revision?: unknown } + return Number.isInteger(parsed.revision) && (parsed.revision as number) >= 0 + ? { revision: parsed.revision as number } + : {} + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + return {} + } + } + + private async bindWorkspace(runId: string, workspaceId: string): Promise { + return this.withRunTransition(runId, async () => { + const run = this.activeRuns.get(runId)?.run ?? await this.read(runId) + return run ? this.bindWorkspaceCurrent(run, workspaceId) : undefined + }) + } + + private async bindWorkspaceCurrent(run: WorkflowRun, workspaceId: string): Promise { + const requested = this.options.workspaceManager.get(workspaceId) + const selection = run.worktreeSelection + if (run.workspaceId === workspaceId) { + if (selection?.policy.mode === "current" && selection.sourceWorkspaceId !== workspaceId) { + const prior = this.cloneRun(run) + selection.sourceWorkspaceId = workspaceId + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + return run + } + if (selection && selection.policy.mode !== "current" && requested + && requested.status === "ready" + && (selection.sourceWorkspaceId === workspaceId || ( + requested.lineageId === selection.sourceWorkspaceLineageId + && this.samePath(requested.path, selection.sourceWorkspacePath) + ))) { + if (selection.sourceWorkspaceId !== workspaceId) { + const prior = this.cloneRun(run) + selection.sourceWorkspaceId = workspaceId + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + } + return run + } + if (["running", "pausing"].includes(run.status)) return undefined + const workspace = requested + if (!workspace || !workspace.lineageId || run.workspaceLineageId !== workspace.lineageId) return undefined + if (!this.samePath(workspace.path, run.workspacePath)) return undefined + const prior = this.cloneRun(run) + const previousId = run.workspaceId + run.workspaceId = workspaceId + if (selection) { + selection.workspaceId = workspaceId + selection.directory = workspace.path + if (selection.policy.mode === "current") selection.sourceWorkspaceId = workspaceId + } + run.workspacePath = workspace.path + try { + await this.persist(run, false) + } catch (error) { + this.restoreRun(run, prior) + throw error + } + if (this.activeWorkspaces.get(previousId) === run.id) this.activeWorkspaces.delete(previousId) + if (holdsWorkflowReservation(run)) this.activeWorkspaces.set(workspaceId, run.id) + return run + } + + private samePath(left: string, right: string): boolean { + const normalizedLeft = path.resolve(left) + const normalizedRight = path.resolve(right) + return process.platform === "win32" + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight + } + + private pathKey(value: string): string { + const resolved = path.resolve(value) + return process.platform === "win32" ? resolved.toLowerCase() : resolved + } + + private matchesCanonicalWorkspace(workspace: WorkspaceDescriptor, lineageId: string, workspacePath: string): boolean { + return workspace.lineageId === lineageId && this.samePath(workspace.path, workspacePath) + } + + private matchesIdentity(identity: WorkflowWorkspaceIdentity, id: string, lineageId: string, workspacePath: string): boolean { + return Boolean( + (identity.id && identity.id === id) + || (identity.lineageId && identity.lineageId === lineageId) + || (identity.path && this.samePath(identity.path, workspacePath)), + ) + } + + private async isWorkspaceWorkflowOwnedCurrent(identity: WorkflowWorkspaceIdentity): Promise { + if (this.admissionFailure) return true + if (identity.lineageId && this.reservedLineages.has(identity.lineageId)) return true + if (identity.path && this.reservedPaths.has(this.pathKey(identity.path))) return true + return Array.from(this.runIndex.values()).some((run) => { + if (!this.indexHoldsReservation(run)) return false + if (this.matchesIdentity(identity, run.workspaceId, run.workspaceLineageId, run.workspacePath)) return true + return Boolean(run.sourceWorkspaceId && run.sourceWorkspaceLineageId && run.sourceWorkspacePath && this.matchesIdentity( + identity, + run.sourceWorkspaceId, + run.sourceWorkspaceLineageId, + run.sourceWorkspacePath, + )) + }) + } + + private async isWorktreeWorkflowOwnedCurrent( + source: WorkflowWorkspaceIdentity, + worktree: WorkflowWorktreeIdentity, + ): Promise { + if (this.admissionFailure) return true + if (this.quarantinedWorktrees.some((run) => { + if (worktree.path && run.worktreeDirectory && this.samePath(run.worktreeDirectory, worktree.path)) return true + if (!worktree.slug || run.worktreeSlug !== worktree.slug) return false + return Boolean( + (source.id && run.sourceWorkspaceId === source.id) + || (source.lineageId && run.sourceWorkspaceLineageId === source.lineageId) + || (source.path && run.sourceWorkspacePath && this.samePath(run.sourceWorkspacePath, source.path)), + ) + })) return true + return Array.from(this.runIndex.values()).some((run) => { + if (!this.indexHoldsReservation(run) || !run.worktreeDirectory) return false + if (worktree.path && this.samePath(run.worktreeDirectory, worktree.path)) return true + if (!run.sourceWorkspaceId || !run.sourceWorkspaceLineageId || !run.sourceWorkspacePath || !this.matchesIdentity( + source, + run.sourceWorkspaceId, + run.sourceWorkspaceLineageId, + run.sourceWorkspacePath, + )) return false + return Boolean(worktree.slug && run.worktreeSlug === worktree.slug) + }) + } + + private persistedAmbiguousSessionIds(run: WorkflowRun): Set { + if (run.definitionSnapshot) return new Set((run.executionNodes ?? []) + .filter((node) => !["completed", "skipped", "failed", "cancelled"].includes(node.status)) + .flatMap((node) => node.sessionIds ?? [])) + return new Set(run.steps + .filter((step) => step.sessionId && (step.status === "running" || step.status === "failed")) + .map((step) => step.sessionId!)) + } + + private hasUnconfirmedAdmittedAction(run: WorkflowRun): boolean { + if (!run.definitionSnapshot) return run.steps.some((step) => step.status === "running") + return Boolean(run.executionNodes?.some((node) => + (node.type === "agent" || node.type === "shell") + && node.attempt > 0 + && !isConfirmedRetryCheckpoint(node) + && !["completed", "skipped", "failed", "cancelled"].includes(node.status))) + } + + private clearPersistedSessions(run: WorkflowRun, sessionIds: Set): void { + for (const node of run.executionNodes ?? []) { + if (!node.sessionIds) continue + node.sessionIds = node.sessionIds.filter((sessionId) => !sessionIds.has(sessionId)) + if (node.sessionIds.length === 0) delete node.sessionIds + } + for (const step of run.steps) if (step.sessionId && sessionIds.has(step.sessionId)) delete step.sessionId + if (run.sessionBindings) { + for (const [key, sessionId] of Object.entries(run.sessionBindings)) { + if (sessionIds.has(sessionId)) delete run.sessionBindings[key] + } + if (Object.keys(run.sessionBindings).length === 0) delete run.sessionBindings + } + } + + private async confirmPersistedSessionAborts(run: WorkflowRun, client: OpencodeClient, action: string): Promise { + const sessionIds = this.persistedAmbiguousSessionIds(run) + if (sessionIds.size === 0) return + const confirmed = await Promise.all(Array.from(sessionIds).map((sessionId) => + this.abortSessionRequest(client, run.id, sessionId))) + if (confirmed.some((result) => !result)) { + const message = `${action} could not confirm every persisted session abort` + await this.persistMutation(run, () => markWorkflowRecoveryRequired(run, message)) + throw new WorkflowRunError(message, 409) + } + this.clearPersistedSessions(run, sessionIds) + } + + private async read(runId: string): Promise { + try { + const run = JSON.parse(await fs.readFile(this.runPath(runId), "utf8")) as WorkflowRun + validatePersistedWorkflowRun(run, runId) + this.indexRun(run) + return run + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } + } + + private async readStoredRun(runId: string): Promise { + try { + const run = JSON.parse(await fs.readFile(this.runPath(runId), "utf8")) as WorkflowRun + validatePersistedWorkflowRun(run, runId) + return run + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined + throw error + } + } + + private async readForListing(entry: string): Promise { + try { + return await this.read(entry.slice(0, -5)) + } catch (error) { + this.options.logger.warn({ err: error, file: entry }, "Skipping corrupt workflow run") + return undefined + } + } + + private runIndexEntry(run: WorkflowRun): RunIndexEntry { + const selection = run.worktreeSelection + return { + id: run.id, + workspaceId: run.workspaceId, + workspaceLineageId: run.workspaceLineageId, + workspacePath: run.workspacePath, + ...(selection ? { + sourceWorkspaceId: selection.sourceWorkspaceId, + sourceWorkspaceLineageId: selection.sourceWorkspaceLineageId, + sourceWorkspacePath: selection.sourceWorkspacePath, + worktreeDirectory: selection.directory, + ...(selection.slug ? { worktreeSlug: selection.slug } : {}), + } : {}), + status: run.status, + createdAt: run.createdAt, + updatedAt: run.updatedAt, + ambiguousSessions: Boolean(run.definitionSnapshot && this.persistedAmbiguousSessionIds(run).size > 0), + } + } + + private indexRun(run: WorkflowRun): void { + this.runIndex.set(run.id, this.runIndexEntry(run)) + } + + private async readRunMetadata(entry: string): Promise { + const runId = entry.slice(0, -5) + try { + const [stored, stat] = await Promise.all([ + fs.readFile(this.runMetadataPath(runId), "utf8"), + fs.stat(path.join(this.options.storageDir, entry)), + ]) + const metadata = JSON.parse(stored) as RunFileMetadata + if (metadata.id !== runId || metadata.size !== stat.size || metadata.mtimeMs !== stat.mtimeMs + || typeof metadata.workspaceId !== "string" || typeof metadata.workspaceLineageId !== "string" + || typeof metadata.workspacePath !== "string" || typeof metadata.createdAt !== "string" + || typeof metadata.updatedAt !== "string" || typeof metadata.ambiguousSessions !== "boolean" + || !["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "completed", "failed", "cancelled", "interrupted", "recovery_required"] + .includes(metadata.status)) return undefined + return metadata + } catch { + return undefined + } + } + + private async writeRunMetadata(run: WorkflowRun): Promise { + const stat = await fs.stat(this.runPath(run.id)) + const metadata: RunFileMetadata = { ...this.runIndexEntry(run), size: stat.size, mtimeMs: stat.mtimeMs } + const destination = this.runMetadataPath(run.id) + const temporary = `${destination}.${randomUUID()}.tmp` + await fs.writeFile(temporary, JSON.stringify(metadata), "utf8") + await fs.rename(temporary, destination) + } + + private indexMatchesWorkspace(entry: RunIndexEntry, workspaceId: string, workspace?: WorkspaceDescriptor): boolean { + if (entry.workspaceId === workspaceId || entry.sourceWorkspaceId === workspaceId) return true + if (!workspace?.lineageId) return false + return this.matchesCanonicalWorkspace(workspace, entry.workspaceLineageId, entry.workspacePath) + || Boolean(entry.sourceWorkspaceLineageId && entry.sourceWorkspacePath + && this.matchesCanonicalWorkspace(workspace, entry.sourceWorkspaceLineageId, entry.sourceWorkspacePath)) + } + + private indexHoldsReservation(entry: RunIndexEntry): boolean { + return ["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"] + .includes(entry.status) + } + + private async refreshRunIndex(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.options.storageDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + this.runIndex.clear() + this.activeWorkspaces.clear() + this.reservedLineages.clear() + this.reservedPaths.clear() + this.quarantinedWorktrees.length = 0 + for (const entry of entries.filter((candidate) => candidate.endsWith(".json"))) { + const runId = entry.slice(0, -5) + try { + const run = await this.readStoredRun(runId) + if (!run) continue + this.indexRun(run) + if (holdsWorkflowReservation(run)) this.reserveRun(run) + } catch (error) { + await this.quarantineMalformedActiveRun(entry) + this.options.logger.error({ err: error, file: entry }, "Failed to refresh workflow ownership") + } + } + } + + private async recoverInterruptedRuns(): Promise { + let entries: string[] + try { + entries = await fs.readdir(this.options.storageDir) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + + for (const entry of entries.filter((candidate) => candidate.endsWith(".json"))) { + let validRun = false + try { + const runId = entry.slice(0, -5) + const recoveryMarker = await this.readRecoveryMarker(runId) + const metadata = await this.readRunMetadata(entry) + if (metadata && !recoveryMarker && !this.indexHoldsReservation(metadata) && !metadata.ambiguousSessions) { + this.runIndex.set(metadata.id, metadata) + continue + } + const run = await this.read(runId) + if (!run) continue + validRun = true + if (this.activeRuns.has(run.id)) { + this.reserveRun(run) + continue + } + if (this.isLeaseLive(run)) { + this.reserveRun(run) + continue + } + const hadStaleLease = Boolean(run.executorLease) + delete run.executorLease + const recoveryMarked = Boolean(recoveryMarker + && (recoveryMarker.revision === undefined || (run.revision ?? 0) <= recoveryMarker.revision)) + if (recoveryMarker && !recoveryMarked) await durableRemove(this.recoveryMarkerPath(run.id)).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to clear stale workflow recovery marker") + }) + await this.writeRunMetadata(run).catch((error) => { + this.options.logger.warn({ err: error, runId: run.id }, "Failed to update workflow run metadata") + }) + if (recoveryMarked) { + const message = "Workflow terminal state was not durably persisted; manual recovery is required and prior gate state cannot be reused" + markWorkflowRecoveryRequired(run, message) + delete run.activeStepId + this.reserveRun(run) + try { + await this.persist(run) + await fs.rm(this.recoveryMarkerPath(run.id), { force: true }) + } catch (error) { + this.admissionFailure = `Workflow admission is blocked because marked recovery state could not be persisted for run ${run.id}; repair storage and restart CodeNomad` + throw error + } + continue + } + if (!holdsWorkflowReservation(run)) continue + const sessionBearing = this.persistedAmbiguousSessionIds(run).size > 0 + const interruptedAction = Boolean(run.definitionSnapshot && run.executionNodes?.some((node) => + node.status === "running" && node.attempt > 0 && (node.type === "agent" || node.type === "shell"))) + const legacyAction = !run.definitionSnapshot && run.steps.some((step) => step.status === "running") + const wasExecuting = run.status === "running" || run.status === "pausing" + const ambiguous = sessionBearing || interruptedAction || legacyAction + if (!wasExecuting && ["interrupted", "recovery_required"].includes(run.status)) { + this.reserveRun(run) + if (hadStaleLease) await this.persist(run) + continue + } + if (!wasExecuting && !ambiguous) { + this.reserveRun(run) + if (hadStaleLease) await this.persist(run) + continue + } + run.error = "CodeNomad restarted before this workflow completed" + if (ambiguous) markWorkflowRecoveryRequired(run, run.error) + else run.status = "interrupted" + const step = run.steps.find((candidate) => candidate.status === "running") + if (step) { + step.status = "failed" + step.error = run.error + step.completedAt = new Date().toISOString() + } + for (const node of run.executionNodes ?? []) { + if ((node.sessionIds?.length && !["completed", "skipped", "failed", "cancelled"].includes(node.status)) + || (node.status === "running" && node.attempt > 0 && (node.type === "agent" || node.type === "shell"))) { + node.status = "interrupted" + node.error = run.error + node.completedAt = new Date().toISOString() + } else if (node.status === "running") { + node.status = "waiting" + } + } + if (run.status === "recovery_required" && this.persistedAmbiguousSessionIds(run).size === 0) { + run.error = "CodeNomad restarted during an action, but no session ID was persisted; termination cannot be positively confirmed and the action will not be repeated" + for (const node of run.executionNodes ?? []) if (node.status === "interrupted") node.error = run.error + if (step) step.error = run.error + } + this.reserveRun(run) + delete run.activeStepId + await this.persist(run) + } catch (error) { + if (validRun) throw error + await this.quarantineMalformedActiveRun(entry) + this.options.logger.error({ err: error, file: entry }, "Failed to recover workflow run") + } + } + await this.pruneHistoryEntries().catch((error) => { + this.options.logger.warn({ err: error }, "Failed to prune global workflow history") + }) + } + + private async quarantineMalformedActiveRun(entry: string): Promise { + let value: unknown + try { + value = JSON.parse(await fs.readFile(path.join(this.options.storageDir, entry), "utf8")) + } catch { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry}; remove or repair it and restart CodeNomad` + return + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry}; remove or repair it and restart CodeNomad` + return + } + const candidate = value as Record + if (["completed", "failed", "cancelled"].includes(candidate.status as string)) return + const quarantineId = typeof candidate.id === "string" && candidate.id ? candidate.id : `malformed:${entry}` + let identifiable = false + if (typeof candidate.workspaceLineageId === "string" && candidate.workspaceLineageId) { + this.reservedLineages.set(candidate.workspaceLineageId, quarantineId) + identifiable = true + } + if (typeof candidate.workspacePath === "string" && candidate.workspacePath) { + this.reservedPaths.set(this.pathKey(candidate.workspacePath), quarantineId) + identifiable = true + } + if (typeof candidate.workspaceId === "string" && candidate.workspaceId) { + this.activeWorkspaces.set(candidate.workspaceId, quarantineId) + } + const selection = candidate.worktreeSelection + if (selection && typeof selection === "object" && !Array.isArray(selection)) { + const worktree = selection as Record + const retained: Partial = {} + for (const [field, value] of [ + ["sourceWorkspaceId", worktree.sourceWorkspaceId], + ["sourceWorkspaceLineageId", worktree.sourceWorkspaceLineageId], + ["sourceWorkspacePath", worktree.sourceWorkspacePath], + ["worktreeDirectory", worktree.directory], + ["worktreeSlug", worktree.slug], + ] as const) if (typeof value === "string" && value) retained[field] = value + if (retained.worktreeDirectory || (retained.worktreeSlug + && (retained.sourceWorkspaceId || retained.sourceWorkspaceLineageId || retained.sourceWorkspacePath))) { + this.quarantinedWorktrees.push(retained) + identifiable = true + } + } + if (!identifiable) { + this.admissionFailure = `Workflow admission is blocked by malformed active run ${entry} without a usable lineage or path; remove or repair it and restart CodeNomad` + } + } + + private async persist(run: WorkflowRun, touch = true): Promise { + const active = this.activeRuns.get(run.id) + if (active?.run === run && active.leaseLost) throw new WorkflowRunError("Workflow executor ownership was lost", 409) + const expectedLease = active?.run === run && !active.leaseDurablyReleased + ? { ownerToken: this.managerToken, fence: active.leaseFence } + : undefined + if (run.executorLease?.ownerToken === this.managerToken) { + run.executorLease.heartbeatAt = new Date().toISOString() + run.executorLease.expiresAt = this.executorLeaseExpiry(run.executorLease.heartbeatAt) + } + if (touch) { + run.updatedAt = new Date().toISOString() + run.revision = (run.revision ?? 0) + 1 + } + validatePersistedWorkflowRun(run, run.id) + const snapshot = JSON.parse(JSON.stringify(run)) as WorkflowRun + const previous = this.persistQueues.get(run.id) ?? Promise.resolve() + const queued = previous.catch(() => undefined).then(async () => { + await ensureDurableDirectory(this.options.storageDir) + const destination = this.runPath(run.id) + await withFilesystemLock(this.runLockPath(run.id), async (assertOwned) => { + const current = await this.readStoredRun(run.id) + if (current) { + if (expectedLease) { + const ownsCurrent = this.isLeaseLive(current) + && current.executorLease?.ownerToken === expectedLease.ownerToken + && current.executorLease.fence === expectedLease.fence + const claimsNextFence = !this.isLeaseLive(current) + && snapshot.executorLease?.ownerToken === expectedLease.ownerToken + && snapshot.executorLease.fence === expectedLease.fence + && expectedLease.fence === (current.executorFence ?? 0) + 1 + if (!ownsCurrent && !claimsNextFence) { + if (active) active.leaseLost = true + throw new WorkflowRunError("Workflow executor ownership was lost", 409) + } + } else { + this.assertNoLiveForeignLease(current) + } + const expectedRevision = touch ? (snapshot.revision ?? 0) - 1 : snapshot.revision ?? 0 + if ((current.revision ?? 0) !== expectedRevision) { + if (expectedLease && active) active.leaseLost = true + throw new WorkflowRunError("Workflow run changed on another CodeNomad host", 409) + } + } else if (!expectedLease || (snapshot.revision ?? 0) !== 1) { + throw new WorkflowRunError("Workflow run journal disappeared", 409) + } + if (snapshot.executorLease?.ownerToken === this.managerToken) { + snapshot.executorLease.heartbeatAt = new Date().toISOString() + snapshot.executorLease.expiresAt = this.executorLeaseExpiry(snapshot.executorLease.heartbeatAt) + if (run.executorLease?.fence === snapshot.executorLease.fence) { + run.executorLease.heartbeatAt = snapshot.executorLease.heartbeatAt + run.executorLease.expiresAt = snapshot.executorLease.expiresAt + } + } + await durableAtomicWrite(destination, `${JSON.stringify(snapshot, null, 2)}\n`, assertOwned) + if (active?.run === run && !snapshot.executorLease) active.leaseDurablyReleased = true + await this.writeRunMetadata(snapshot).catch((error) => { + this.options.logger.warn({ err: error, runId: snapshot.id }, "Failed to update workflow run metadata") + }) + }) + this.indexRun(snapshot) + const workspaceIds = new Set([snapshot.workspaceId, snapshot.worktreeSelection?.sourceWorkspaceId].filter(Boolean) as string[]) + for (const instanceId of workspaceIds) this.options.eventBus.publish({ + type: "instance.event", + instanceId, + // ponytail: checkpoint events stay compact; clients fetch full state at user-visible boundaries. + event: { type: "workflow.run.updated", properties: { + runId: snapshot.id, revision: snapshot.revision, status: snapshot.status, updatedAt: snapshot.updatedAt, + } }, + }) + if (["completed", "failed", "cancelled"].includes(snapshot.status)) { + await this.pruneHistory(snapshot.worktreeSelection?.sourceWorkspaceLineageId ?? snapshot.workspaceLineageId).catch((error) => { + this.options.logger.warn({ err: error, runId: snapshot.id }, "Failed to prune workflow history") + }) + } + }) + this.persistQueues.set(run.id, queued) + try { + await queued + } finally { + if (this.persistQueues.get(run.id) === queued) this.persistQueues.delete(run.id) + } + } + + private async renewExecutorLease(active: ActiveRun): Promise { + await (this.persistQueues.get(active.run.id) ?? Promise.resolve()).catch(() => undefined) + if (active.run.executorLease?.ownerToken !== this.managerToken + || active.run.executorLease.fence !== active.leaseFence) return + await withFilesystemLock(this.runLockPath(active.run.id), async (assertOwned) => { + const current = await this.readStoredRun(active.run.id) + if (active.run.executorLease?.ownerToken !== this.managerToken + || active.run.executorLease.fence !== active.leaseFence) return + if (current?.executorLease?.ownerToken !== this.managerToken + || current.executorLease.fence !== active.leaseFence + || !this.isLeaseLive(current)) { + throw new WorkflowRunError("Workflow executor ownership was lost", 409) + } + current.executorLease.heartbeatAt = new Date().toISOString() + current.executorLease.expiresAt = this.executorLeaseExpiry(current.executorLease.heartbeatAt) + await durableAtomicWrite(this.runPath(current.id), `${JSON.stringify(current, null, 2)}\n`, assertOwned) + if (active.run.executorLease?.fence === active.leaseFence) { + active.run.executorLease.heartbeatAt = current.executorLease.heartbeatAt + active.run.executorLease.expiresAt = current.executorLease.expiresAt + } + }) + } + + private async revalidateExecutorFence(active: ActiveRun): Promise { + await (this.persistQueues.get(active.run.id) ?? Promise.resolve()) + this.throwIfCancelled(active) + try { + await withFilesystemLock(this.runLockPath(active.run.id), async (assertOwned) => { + const current = await this.readStoredRun(active.run.id) + await assertOwned() + if (!current || !["running", "pausing", "waiting_for_review", "waiting_for_input"].includes(current.status) + || !this.isLeaseLive(current) + || current.executorLease?.ownerToken !== this.managerToken + || current.executorLease.fence !== active.leaseFence) { + throw new WorkflowRunError("Workflow executor ownership was lost", 409) + } + }) + } catch (error) { + active.leaseLost = true + active.releaseBlocked = true + active.cancelRequested = true + active.abortController.abort(new WorkflowCancelledError("Workflow executor lease was lost")) + throw error + } + } + + private async abandonExecutorLease(active: ActiveRun): Promise { + await withFilesystemLock(this.runLockPath(active.run.id), async (assertOwned) => { + const current = await this.readStoredRun(active.run.id) + if (!current) return + if (current.executorLease?.ownerToken !== this.managerToken + || current.executorLease.fence !== active.leaseFence) { + throw new WorkflowRunError("Workflow executor ownership was lost", 409) + } + delete current.executorLease + await durableAtomicWrite(this.runPath(current.id), `${JSON.stringify(current, null, 2)}\n`, assertOwned) + active.leaseDurablyReleased = true + }) + } + + private async pruneHistory(workspaceLineageId: string): Promise { + return this.pruneHistoryEntries(workspaceLineageId) + } + + private async pruneHistoryEntries(workspaceLineageId?: string): Promise { + const terminal = Array.from(this.runIndex.values()) + .filter((run) => ["completed", "failed", "cancelled"].includes(run.status)) + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)) + const lineage = workspaceLineageId + ? terminal.filter((run) => (run.sourceWorkspaceLineageId ?? run.workspaceLineageId) === workspaceLineageId) + : [] + const expired = new Set([ + ...terminal.slice(WORKFLOW_HISTORY_LIMIT).map((run) => run.id), + ...lineage.slice(WORKFLOW_HISTORY_LIMIT).map((run) => run.id), + ]) + await Promise.all(Array.from(expired).map(async (runId) => { + await Promise.all([fs.rm(this.runPath(runId), { force: true }), fs.rm(this.runMetadataPath(runId), { force: true })]) + this.runIndex.delete(runId) + })) + } +} + +function currentProcessOwner(): { hostname: string; pid: number; processStart?: string; bootId?: string } { + const snapshot = process.platform === "win32" + ? probeWindowsProcesses(spawnSync, 1_000) + : probePosixProcesses(spawnSync, 1_000, process.platform, { pids: [process.pid] }) + const identity = snapshot.ok ? snapshot.processes.get(process.pid) : undefined + return { + hostname: os.hostname(), pid: process.pid, + ...(identity ? { processStart: identity.startTime, ...(identity.bootId ? { bootId: identity.bootId } : {}) } : {}), + } +} + +function executorProcessLiveness(pid: number, processStart?: string, bootId?: string): "alive" | "dead" | "unknown" { + try { + process.kill(pid, 0) + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH" ? "dead" : "unknown" + } + if (!processStart) return "alive" + const snapshot = process.platform === "win32" + ? probeWindowsProcesses(spawnSync, 1_000) + : probePosixProcesses(spawnSync, 1_000, process.platform, { pids: [pid] }) + if (snapshot.ok) { + const current = snapshot.processes.get(pid) + if (!current) return "dead" + return current.startTime === processStart && (!bootId || current.bootId === bootId) ? "alive" : "dead" + } + return "unknown" +} + +async function durableAtomicWrite(destination: string, contents: string, assertOwned?: () => Promise): Promise { + const temporary = `${destination}.${randomUUID()}.tmp` + try { + const handle = await fs.open(temporary, "wx") + try { + await handle.writeFile(contents, "utf8") + await handle.sync() + } finally { + await handle.close() + } + await assertOwned?.() + await fs.rename(temporary, destination) + await assertOwned?.() + await syncDirectory(path.dirname(destination)) + } catch (error) { + await fs.rm(temporary, { force: true }).catch(() => undefined) + throw error + } +} + +async function durableRemove(destination: string): Promise { + await fs.rm(destination, { force: true }) + await syncDirectory(path.dirname(destination)) +} + +async function syncDirectory(directoryPath: string): Promise { + let directory: fs.FileHandle | undefined + try { + directory = await fs.open(directoryPath, "r") + await directory.sync() + } catch (error) { + if (!["EINVAL", "ENOTSUP", "EISDIR", "EPERM", "EBADF", "ENOSYS"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error + } finally { + await directory?.close() + } +} + +async function ensureDurableDirectory(directoryPath: string): Promise { + const created = await fs.mkdir(directoryPath, { recursive: true }) + if (!created) return + const firstCreated = path.resolve(created) + let current = path.resolve(directoryPath) + while (true) { + await syncDirectory(path.dirname(current)) + if (current === firstCreated) return + const parent = path.dirname(current) + if (parent === current) return + current = parent + } +} diff --git a/packages/server/src/workflows/run-state.ts b/packages/server/src/workflows/run-state.ts new file mode 100644 index 000000000..ba698f420 --- /dev/null +++ b/packages/server/src/workflows/run-state.ts @@ -0,0 +1,208 @@ +import type { + WorkflowDefinitionV1, + WorkflowDefinitionRecord, + WorkflowDefinitionRunCreateRequest, + WorkflowExecutionNode, + WorkflowNode, + WorkflowRun, + WorkflowUsage, +} from "../api-types" +import { parseWorkflowDefinition, WORKFLOW_LIMITS } from "./definition-schema" + +const clone = (value: T): T => JSON.parse(JSON.stringify(value)) as T + +export const emptyWorkflowUsage = (): WorkflowUsage => ({ + cost: 0, + tokens: 0, + inputTokens: 0, + outputTokens: 0, + reasoningTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, +}) + +export const definitionRunFields = ( + record: WorkflowDefinitionRecord, + input: WorkflowDefinitionRunCreateRequest, +): Pick => ({ + definitionId: record.id, + definitionRevision: record.revision, + definitionSnapshot: clone(record.definition), + inputs: clone(input.inputs ?? {}), + executionNodes: [], + usage: emptyWorkflowUsage(), +}) + +export const holdsWorkflowReservation = (run: WorkflowRun) => + ["running", "pausing", "paused", "waiting_for_review", "waiting_for_input", "interrupted", "recovery_required"].includes(run.status) + +export const isConfirmedRetryCheckpoint = (node: WorkflowExecutionNode) => + (node.type === "agent" || node.type === "shell") && node.status === "waiting" + && node.attempt > 0 && !node.sessionIds?.length + +export function markWorkflowRecoveryRequired(run: WorkflowRun, message: string) { + run.status = "recovery_required" + run.error = message + for (const node of run.executionNodes ?? []) { + if (node.status !== "running" && node.status !== "waiting" + && !(node.sessionIds?.length && !["completed", "skipped", "failed", "cancelled"].includes(node.status))) continue + node.status = "interrupted" + node.error = message + node.completedAt = new Date().toISOString() + } + run.pauseRequested = false + delete run.pendingGate +} + +const RUN_STATUSES = new Set([ + "running", "pausing", "paused", "waiting_for_review", "waiting_for_input", + "completed", "failed", "cancelled", "interrupted", "recovery_required", +]) +const EXECUTION_STATUSES = new Set(["pending", "running", "waiting", "completed", "skipped", "failed", "cancelled", "interrupted"]) +const EXECUTION_TYPES = new Set(["sequence", "parallel", "foreach", "repeat", "agent", "shell", "gate", "condition", "workflow"]) +const isNonEmptyString = (value: unknown): value is string => typeof value === "string" && value.length > 0 +const SESSION_KEY_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/ +const isTimestamp = (value: unknown): value is string => isNonEmptyString(value) && Number.isFinite(Date.parse(value)) +const MAX_EXECUTOR_LEASE_MS = 30_000 +const MAX_CLOCK_SKEW_MS = 60_000 +const validUsage = (usage: WorkflowUsage) => Object.values(usage).every((value) => + typeof value === "number" && Number.isFinite(value) && value >= 0) + +export function validatePersistedWorkflowRun(value: unknown, runId: string): asserts value is WorkflowRun { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`Invalid stored workflow run ${runId}`) + const run = value as WorkflowRun + if (run.id !== runId + || !isNonEmptyString(run.workspaceId) + || !isNonEmptyString(run.workspaceLineageId) + || !isNonEmptyString(run.workspacePath) + || !isNonEmptyString(run.objective) + || !RUN_STATUSES.has(run.status) + || !isTimestamp(run.createdAt) + || !isTimestamp(run.updatedAt) + || !Array.isArray(run.steps)) { + throw new Error(`Invalid stored workflow run ${runId}`) + } + if (run.usage && !validUsage(run.usage)) throw new Error(`Invalid workflow usage for workflow run ${runId}`) + if (run.executorFence !== undefined && (!Number.isInteger(run.executorFence) || run.executorFence < 1)) { + throw new Error(`Invalid executor fence for workflow run ${runId}`) + } + if (run.executorLease && ( + !isNonEmptyString(run.executorLease.ownerToken) + || !Number.isInteger(run.executorLease.fence) || run.executorLease.fence < 1 + || run.executorLease.fence !== run.executorFence + || !isTimestamp(run.executorLease.heartbeatAt) + || !isTimestamp(run.executorLease.expiresAt) + || Date.parse(run.executorLease.heartbeatAt) > Date.now() + MAX_CLOCK_SKEW_MS + || Date.parse(run.executorLease.expiresAt) <= Date.parse(run.executorLease.heartbeatAt) + || Date.parse(run.executorLease.expiresAt) - Date.parse(run.executorLease.heartbeatAt) > MAX_EXECUTOR_LEASE_MS + || ((run.executorLease.hostname !== undefined || run.executorLease.pid !== undefined + || run.executorLease.processStart !== undefined || run.executorLease.bootId !== undefined) && ( + !isNonEmptyString(run.executorLease.hostname) + || !Number.isInteger(run.executorLease.pid) || run.executorLease.pid! < 1 + || (run.executorLease.processStart !== undefined && !isNonEmptyString(run.executorLease.processStart)) + || (run.executorLease.bootId !== undefined && !isNonEmptyString(run.executorLease.bootId)) + )) + )) throw new Error(`Invalid executor lease for workflow run ${runId}`) + if (run.sessionBindings !== undefined && ( + !run.sessionBindings || typeof run.sessionBindings !== "object" || Array.isArray(run.sessionBindings) + || Object.entries(run.sessionBindings).length > WORKFLOW_LIMITS.expandedNodes + || Object.entries(run.sessionBindings).some(([key, sessionId]) => + key.length > 100 || !SESSION_KEY_PATTERN.test(key) || !isNonEmptyString(sessionId) || sessionId.length > 200) + )) throw new Error(`Invalid session bindings for workflow run ${runId}`) + if (!run.definitionSnapshot) return + const { definition } = parseWorkflowDefinition(run.definitionSnapshot) + if (run.definitionId !== definition.id || !Number.isInteger(run.definitionRevision) || run.definitionRevision! < 1) { + throw new Error(`Invalid definition snapshot for workflow run ${runId}`) + } + const expandedLimit = definition.maxExpandedNodes ?? WORKFLOW_LIMITS.expandedNodes + if (!Array.isArray(run.executionNodes) || run.executionNodes.length > expandedLimit) { + throw new Error(`Invalid execution nodes for workflow run ${runId}`) + } + if (Object.keys(run.sessionBindings ?? {}).length > expandedLimit) { + throw new Error(`Too many session bindings for workflow run ${runId}`) + } + if (new Set(run.executionNodes.map((node) => node.id)).size !== run.executionNodes.length + || new Set(run.executionNodes.map((node) => node.instanceKey)).size !== run.executionNodes.length) { + throw new Error(`Duplicate execution nodes for workflow run ${runId}`) + } + for (const node of run.executionNodes) { + if (!isNonEmptyString(node.id) || !isNonEmptyString(node.instanceKey) || !isNonEmptyString(node.definitionNodeId) + || (node.definitionInvocationKey !== undefined && !isNonEmptyString(node.definitionInvocationKey)) + || !EXECUTION_TYPES.has(node.type) || !EXECUTION_STATUSES.has(node.status) || !Number.isInteger(node.attempt) || node.attempt < 0 + || (node.sessionIds !== undefined && (!Array.isArray(node.sessionIds) || node.sessionIds.some((id) => !isNonEmptyString(id)))) + || (node.usage && !validUsage(node.usage))) { + throw new Error(`Invalid execution node for workflow run ${runId}`) + } + } + const snapshots = run.savedDefinitionSnapshots ?? [] + if (snapshots.length > WORKFLOW_LIMITS.staticNodes) throw new Error(`Too many saved definition snapshots for workflow run ${runId}`) + const byKey = new Map() + for (const snapshot of snapshots) { + const parsed = parseWorkflowDefinition(snapshot.definition).definition + if (snapshot.id !== parsed.id || !Number.isInteger(snapshot.revision) || snapshot.revision < 1) { + throw new Error(`Invalid saved definition snapshot for workflow run ${runId}`) + } + const key = `${snapshot.id}@${snapshot.revision}` + if (byKey.has(key)) throw new Error(`Duplicate saved definition snapshot for workflow run ${runId}`) + byKey.set(key, parsed) + } + const inspect = (node: typeof definition.root, stack: string[], depth: number): void => { + if (depth > WORKFLOW_LIMITS.nestingDepth) throw new Error(`Saved workflow nesting is too deep for workflow run ${runId}`) + if (node.type === "workflow") { + const key = `${node.definitionId}@${node.definitionRevision}` + const snapshot = byKey.get(key) + if (!snapshot || !node.definitionRevision) throw new Error(`Missing saved definition snapshot for workflow run ${runId}`) + if (stack.includes(snapshot.id)) throw new Error(`Saved workflow cycle in workflow run ${runId}`) + inspect(snapshot.root, [...stack, snapshot.id], depth + 1) + return + } + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) inspect(child, stack, depth) + } + inspect(definition.root, [definition.id], 0) + const typesByDefinition = new Map>() + const collectTypes = (key: string, root: WorkflowNode) => { + const types = new Map() + const visit = (node: WorkflowNode): void => { + types.set(node.id, node.type) + const children = node.type === "sequence" ? node.steps + : node.type === "parallel" ? node.branches + : node.type === "foreach" || node.type === "repeat" ? [node.body] + : node.type === "condition" ? [node.then, ...(node.else ? [node.else] : [])] + : [] + for (const child of children) visit(child) + } + visit(root) + typesByDefinition.set(key, types) + } + const rootKey = `${run.definitionId}@${run.definitionRevision}` + collectTypes(rootKey, definition.root) + for (const [key, snapshot] of byKey) collectTypes(key, snapshot.root) + for (const node of run.executionNodes) { + const segments = node.instanceKey.split("/") + const savedInvocation = segments.map((segment, index) => ({ segment, index })) + .filter(({ segment }) => byKey.has(segment)).at(-1) + const definitionKey = savedInvocation?.segment ?? rootKey + const invocationKey = savedInvocation ? segments.slice(0, savedInvocation.index + 1).join("/") : rootKey + if (node.definitionInvocationKey !== undefined && node.definitionInvocationKey !== invocationKey) { + throw new Error(`Invalid definition invocation scope for workflow run ${runId}`) + } + if (typesByDefinition.get(definitionKey)?.get(node.definitionNodeId) !== node.type) { + throw new Error(`Execution node does not match the pinned graph for workflow run ${runId}`) + } + } + if (run.worktreeSelection) { + const selection = run.worktreeSelection + if (selection.workspaceId !== run.workspaceId || selection.directory !== run.workspacePath + || !selection.sourceWorkspaceId || !selection.sourceWorkspaceLineageId || !selection.sourceWorkspacePath) { + throw new Error(`Invalid worktree selection for workflow run ${runId}`) + } + if (selection.policy.mode !== "current" && selection.policy.slug !== selection.slug) { + throw new Error(`Invalid worktree selection for workflow run ${runId}`) + } + } +} diff --git a/packages/server/src/workflows/runtime.test.ts b/packages/server/src/workflows/runtime.test.ts new file mode 100644 index 000000000..33c0c83f9 --- /dev/null +++ b/packages/server/src/workflows/runtime.test.ts @@ -0,0 +1,2159 @@ +import assert from "node:assert/strict" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { describe, it } from "node:test" +import type { OpencodeClient } from "@opencode-ai/sdk/v2/client" +import type { WorkflowDefinitionV1, WorkflowRun } from "../api-types" +import type { EventBus } from "../events/bus" +import type { Logger } from "../logger" +import type { WorkspaceManager } from "../workspaces/manager" +import { WorkflowInterpreter } from "./interpreter" +import { WorkflowManager } from "./manager" +import { validatePersistedWorkflowRun } from "./run-state" + +const workspaceManager = { + get: (id: string) => ({ id, lineageId: "lineage", path: "C:/workspace", status: "ready" }), + list: () => [{ id: "workspace", lineageId: "lineage", path: "C:/workspace", status: "ready" }], +} as unknown as WorkspaceManager +const eventBus = { publish: () => true } as unknown as EventBus +const logger = { warn() {}, error() {} } as unknown as Logger +const execFileAsync = promisify(execFile) +const usage = (cost = 0.1, tokens = 10) => ({ + role: "assistant", cost, + tokens: { total: tokens, input: tokens, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, +}) +const workflowTools = { ids: async () => ({ data: ["read", "glob", "grep", "lsp", "bash", "shell", "write", "edit", "apply_patch", "task"] }) } +const waitFor = async (manager: WorkflowManager, id: string, statuses: WorkflowRun["status"][]) => { + let latest: WorkflowRun | undefined + for (let attempt = 0; attempt < 600; attempt += 1) { + latest = (await manager.get(id))! + if (statuses.includes(latest.status)) return latest + await new Promise((resolve) => setTimeout(resolve, 5)) + } + throw new Error(`Workflow ${id} did not reach ${statuses.join(", ")}: ${latest?.status} ${latest?.error ?? ""}`) +} + +describe("declarative workflow runtime", () => { + it("runs branch, foreach, parallel and bounded repeat with structured handoff", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-runtime-")) + const prompts: Array> = [] + let sessions = 0 + const client = { + tool: { ids: async () => ({ data: ["read", "shell", "write"] }) }, + session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(input) + const text = JSON.stringify(input.parts) + return { data: text.includes("Seed") + ? { info: { ...usage(), structured: { go: true, items: [1, 2, 3] } }, parts: [] } + : { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + shell: async () => ({ data: { info: usage(), parts: [{ type: "tool", state: { status: "completed", output: "shell" } }] } }), + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + const definition: WorkflowDefinitionV1 = { + version: 1, id: "dynamic", name: "Dynamic", maxConcurrency: 4, + root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "seed", title: "Seed", instructions: "Seed", tools: ["read"], outputSchema: { + type: "object", required: ["go", "items"], properties: { go: { type: "boolean" }, items: { type: "array" } }, + } }, + { type: "condition", id: "branch", condition: { value: { $ref: "nodes.seed.output.go" }, equals: true }, then: { + type: "foreach", id: "each", items: { $ref: "nodes.seed.output.items" }, item: "item", maxItems: 3, maxConcurrency: 2, + body: { type: "agent", id: "worker", instructions: "Handle item", context: { $ref: "vars.item" } }, + } }, + { type: "parallel", id: "parallel", maxConcurrency: 2, branches: [ + { type: "agent", id: "left", instructions: "Left" }, + { type: "shell", id: "right", title: "Right", agent: "build", command: "git status --short" }, + ] }, + { type: "repeat", id: "repeat", maxIterations: 2, body: { type: "agent", id: "again", instructions: "Again" } }, + ] }, + } + try { + const stored = await manager.createDefinition(definition) + const started = await manager.start({ workspaceId: "workspace", definitionId: stored.id, objective: "Run graph" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(run.definitionRevision, 1) + assert.deepEqual(run.definitionSnapshot, definition) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "worker").length, 3) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "again").length, 2) + assert.equal(run.executionNodes?.find((node) => node.definitionNodeId === "right")?.output, "shell") + assert.match(JSON.stringify(prompts), /Context.*1/) + assert.deepEqual((prompts[0]?.format as Record)?.type, "json_schema") + assert.deepEqual(prompts[0]?.tools, { "*": false, read: true, shell: false, write: false }) + assert.equal(run.usage?.tokens, 80) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps repeat references scoped to their foreach instance", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-repeat-scope-")) + let sessions = 0 + let seeds = 0 + let releaseSeeds!: () => void + const bothSeeds = new Promise((resolve) => { releaseSeeds = resolve }) + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `repeat-scope-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + if (prompt.includes("Seed")) { + if (++seeds === 2) releaseSeeds() + await bothSeeds + const item = prompt.includes("Context:\\n1") ? 1 : 0 + return { data: { info: { ...usage(), structured: item }, parts: [] } } + } + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "repeat-scope", name: "Repeat scope", maxConcurrency: 2, root: { + type: "foreach", id: "each", items: [0, 1], item: "item", maxItems: 2, maxConcurrency: 2, body: { + type: "sequence", id: "iteration", steps: [ + { type: "agent", id: "seed", title: "Seed", instructions: "Seed", context: { $ref: "vars.item" }, + outputSchema: { type: "number" } }, + { type: "repeat", id: "repeat", maxIterations: 1, + while: { value: { $ref: "nodes.seed.output" }, equals: { $ref: "vars.item" } }, + body: { type: "agent", id: "work", instructions: "Work" } }, + ], + }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "repeat-scope" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "work").length, 2) + } finally { + releaseSeeds() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("does not resolve a skipped foreach producer from a completed sibling", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-skipped-sibling-")) + let sessions = 0 + const consumerPrompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `skipped-sibling-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + if (prompt.includes("Consume")) consumerPrompts.push(prompt) + return { data: { info: usage(), parts: [{ type: "text", text: "produced" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "skipped-sibling", name: "Skipped sibling", root: { + type: "foreach", id: "each", items: [0, 1], item: "item", maxItems: 2, maxConcurrency: 1, body: { + type: "sequence", id: "iteration", steps: [ + { type: "agent", id: "producer", instructions: "Produce", + if: { value: { $ref: "vars.item" }, equals: 0 } }, + { type: "agent", id: "consumer", title: "Consume", instructions: "Consume", + context: { $ref: "nodes.producer.output" } }, + ], + }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "skipped-sibling" }) + await waitFor(manager, started.id, ["completed"]) + assert.equal(consumerPrompts.length, 2) + assert.match(consumerPrompts[0]!, /Context/) + assert.doesNotMatch(consumerPrompts[1]!, /Context/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("resolves outer nodes inside foreach without crossing iteration siblings", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-outer-scope-")) + let sessions = 0 + const consumerPrompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `outer-scope-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + if (prompt.includes("Outer producer")) { + return { data: { info: { ...usage(), structured: "outer" }, parts: [] } } + } + if (prompt.includes("Iteration producer")) { + const item = prompt.includes("Context:\\n1") ? "one" : "zero" + return { data: { info: { ...usage(), structured: item }, parts: [] } } + } + consumerPrompts.push(prompt) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "outer-scope", name: "Outer scope", root: { + type: "sequence", id: "root", steps: [ + { type: "sequence", id: "setup", steps: [ + { type: "agent", id: "outer", title: "Outer producer", instructions: "Outer producer", outputSchema: { type: "string" } }, + ] }, + { type: "foreach", id: "each", items: [0, 1], item: "item", maxItems: 2, maxConcurrency: 1, body: { + type: "sequence", id: "iteration", steps: [ + { type: "agent", id: "local", title: "Iteration producer", instructions: "Iteration producer", + context: { $ref: "vars.item" }, outputSchema: { type: "string" } }, + { type: "agent", id: "consumer", title: "Consume", instructions: "Consume", context: { + outer: { $ref: "nodes.outer.output" }, local: { $ref: "nodes.local.output" }, + } }, + ], + } }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "outer-scope" }) + await waitFor(manager, started.id, ["completed"]) + assert.equal(consumerPrompts.length, 2) + assert.match(consumerPrompts[0]!, /outer.*zero/) + assert.match(consumerPrompts[1]!, /outer.*one/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("allows shared context references but rejects ancestor cycles", () => { + const shared = { value: 1 } + const run = { + id: "context", workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Context", status: "running", steps: [], executionNodes: [], createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + } as WorkflowRun + const interpreter = new WorkflowInterpreter({ + run, client: {} as OpencodeClient, persist: async () => {}, signal: () => new AbortController().signal, + sessionStarted: () => true, sessionFinished: () => {}, abortSession: async () => true, isCancelled: () => false, + }) + const context = { vars: {}, inputs: { shared }, budgets: [], limiters: [], definitionInvocationKey: "context@1" } + const resolved = (interpreter as any).resolveContext( + [{ $ref: "inputs.shared" }, { $ref: "inputs.shared" }], context, new AbortController().signal, + ) + assert.equal(resolved[0], resolved[1]) + const cyclic: Record = {} + cyclic.self = cyclic + assert.throws(() => (interpreter as any).resolveContext( + { $ref: "inputs.cyclic" }, { ...context, inputs: { cyclic } }, new AbortController().signal, + ), /contains a cycle/) + }) + + it("inherits omitted agent tools and applies any explicit installed-tool allowlist", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-tools-")) + const prompts: Array> = [] + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `tools-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(input) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "tools-off", name: "Tools off", root: { + type: "agent", id: "work", instructions: "Read only", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "tools-off" }) + await waitFor(manager, started.id, ["completed"]) + assert.equal(Object.prototype.hasOwnProperty.call(prompts[0], "tools"), false) + + await manager.createDefinition({ version: 1, id: "tools-dangerous", name: "Dangerous", root: { + type: "agent", id: "work", instructions: "Run", tools: ["bash", "task"], + } }) + const dangerous = await manager.start({ workspaceId: "workspace", definitionId: "tools-dangerous" }) + await waitFor(manager, dangerous.id, ["completed"]) + assert.equal((prompts[1]?.tools as Record).bash, true) + assert.equal((prompts[1]?.tools as Record).task, true) + assert.equal((prompts[1]?.tools as Record).edit, false) + + await manager.createDefinition({ version: 1, id: "tools-missing", name: "Missing", root: { + type: "agent", id: "work", instructions: "Run", tools: ["not-installed"], + } }) + const missing = await manager.start({ workspaceId: "workspace", definitionId: "tools-missing" }) + const failed = await waitFor(manager, missing.id, ["failed"]) + assert.match(failed.error ?? "", /tool not-installed is unavailable/) + assert.equal(prompts.length, 2) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("persists, reuses, and serializes named agent sessions", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-persistent-session-")) + const promptSessions: string[] = [] + let sessions = 0 + let activePrompts = 0 + let maxActivePrompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `persistent-${++sessions}` } }), + prompt: async (input: { sessionID: string }) => { + promptSessions.push(input.sessionID) + maxActivePrompts = Math.max(maxActivePrompts, ++activePrompts) + await new Promise((resolve) => setTimeout(resolve, 10)) + activePrompts-- + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "persistent-session", name: "Persistent session", maxConcurrency: 2, root: { + type: "sequence", id: "root", steps: [ + { type: "repeat", id: "refine", maxIterations: 2, + body: { type: "agent", id: "worker", sessionKey: "luna-worker", instructions: "Refine" } }, + { type: "parallel", id: "review", maxConcurrency: 2, branches: [ + { type: "agent", id: "left", sessionKey: "luna-worker", instructions: "Review left" }, + { type: "agent", id: "right", sessionKey: "luna-worker", instructions: "Review right" }, + ] }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "persistent-session" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(sessions, 2) + assert.deepEqual(promptSessions, Array(4).fill("persistent-2")) + assert.equal(maxActivePrompts, 1) + assert.deepEqual(run.sessionBindings, { "luna-worker": "persistent-2" }) + const persisted = JSON.parse(await fs.readFile(path.join(directory, `${run.id}.json`), "utf8")) as WorkflowRun + validatePersistedWorkflowRun(persisted, run.id) + assert.deepEqual(persisted.sessionBindings, run.sessionBindings) + assert.throws(() => validatePersistedWorkflowRun({ ...persisted, sessionBindings: { "bad key": "session" } }, run.id), + /Invalid session bindings/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("holds a named session until the prior node terminal checkpoint is persisted", async () => { + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "session-handoff", name: "Session handoff", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "left", sessionKey: "worker", instructions: "Left" }, + { type: "agent", id: "right", sessionKey: "worker", instructions: "Right" }, + ] } } + const run = { + id: "session-handoff", workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Handoff", status: "running", steps: [], definitionId: definition.id, definitionRevision: 1, + definitionSnapshot: definition, inputs: {}, executionNodes: [], createdAt: now, updatedAt: now, + } as WorkflowRun + let prompts = 0 + let blockCheckpoint = true + let checkpointEntered!: () => void + let releaseCheckpoint!: () => void + const entered = new Promise((resolve) => { checkpointEntered = resolve }) + const release = new Promise((resolve) => { releaseCheckpoint = resolve }) + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: "shared-session" } }), + prompt: async () => { + prompts++ + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const interpreter = new WorkflowInterpreter({ + run, client, + persist: async () => { + const completed = run.executionNodes?.filter((node) => node.type === "agent" && node.status === "completed").length ?? 0 + if (blockCheckpoint && completed === 1) { + blockCheckpoint = false + checkpointEntered() + await release + } + }, + signal: () => new AbortController().signal, sessionStarted: () => true, sessionFinished: () => {}, + abortSession: async () => true, isCancelled: () => false, + }) + + const execution = interpreter.execute() + await entered + await new Promise((resolve) => setTimeout(resolve, 20)) + assert.equal(prompts, 1) + releaseCheckpoint() + await execution + assert.equal(prompts, 2) + }) + + it("includes named-session permit waiting in the node timeout", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-session-timeout-")) + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: "timed-session" } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 250) + options?.signal?.addEventListener("abort", () => { + clearTimeout(timer) + reject(options.signal!.reason) + }, { once: true }) + }) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "session-timeout", name: "Session timeout", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "left", sessionKey: "worker", instructions: "Left", timeoutMs: 400 }, + { type: "agent", id: "right", sessionKey: "worker", instructions: "Right", timeoutMs: 400 }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "session-timeout" }) + await waitFor(manager, started.id, ["failed"]) + assert.ok(prompts >= 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("restores a named agent session without creating another session", async () => { + const now = new Date().toISOString() + const prompts: string[] = [] + const run = { + id: "restored-session", workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Continue", status: "running", rootSessionId: "root", steps: [], definitionId: "restored", + definitionRevision: 1, definitionSnapshot: { version: 1, id: "restored", name: "Restored", root: { + type: "agent", id: "continue", sessionKey: "worker", instructions: "Continue", + } }, sessionBindings: { worker: "existing-session" }, executionNodes: [], createdAt: now, updatedAt: now, + } as WorkflowRun + const client = { tool: workflowTools, session: { + create: async () => { throw new Error("unexpected session creation") }, + prompt: async (input: { sessionID: string }) => { + prompts.push(input.sessionID) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const checkpoints: WorkflowRun["executionNodes"][] = [] + const interpreter = new WorkflowInterpreter({ + run, client, persist: async () => { checkpoints.push(JSON.parse(JSON.stringify(run.executionNodes))) }, signal: () => new AbortController().signal, + sessionStarted: () => true, sessionFinished: () => {}, abortSession: async () => true, isCancelled: () => false, + }) + + await interpreter.execute() + assert.deepEqual(prompts, ["existing-session"]) + assert.equal(checkpoints.some((nodes) => nodes?.some((node) => node.attempt > 0 && !node.sessionIds?.length)), false) + }) + + it("fails only when a repeat configured to fail exhausts its iterations", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-repeat-exhausted-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `repeat-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "retry" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "repeat-fail", name: "Repeat fail", root: { + type: "repeat", id: "retry", maxIterations: 1, onExhausted: "fail", + body: { type: "agent", id: "work", instructions: "Retry" }, + } }) + const exhausted = await manager.start({ workspaceId: "workspace", definitionId: "repeat-fail" }) + const failed = await waitFor(manager, exhausted.id, ["failed"]) + assert.match(failed.error ?? "", /exhausted 1 iterations/) + + await manager.createDefinition({ version: 1, id: "repeat-stop", name: "Repeat stop", root: { + type: "repeat", id: "retry", maxIterations: 1, while: false, onExhausted: "fail", + body: { type: "agent", id: "work", instructions: "Do not run" }, + } }) + const stopped = await manager.start({ workspaceId: "workspace", definitionId: "repeat-stop" }) + await waitFor(manager, stopped.id, ["completed"]) + + await manager.createDefinition({ version: 1, id: "repeat-satisfied", name: "Repeat satisfied", root: { + type: "repeat", id: "retry", maxIterations: 1, + while: { value: { $ref: "nodes.work.output" }, notEquals: "retry" }, onExhausted: "fail", + body: { type: "agent", id: "work", instructions: "Retry" }, + } }) + const satisfied = await manager.start({ workspaceId: "workspace", definitionId: "repeat-satisfied" }) + await waitFor(manager, satisfied.id, ["completed"]) + assert.equal(sessions, 5) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("pauses at a durable boundary and resumes without repeating completed work", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-")) + let release!: () => void + const first = new Promise((resolve) => { release = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async () => { prompts++; if (prompts === 1) await first; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause", name: "Pause", root: { type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal((await manager.pause(started.id))?.status, "pausing") + release() + while (started.status !== "paused") await new Promise((resolve) => setTimeout(resolve, 1)) + const resumed = manager.resume(started.id) + const racedStart = manager.start({ workspaceId: "workspace", definitionId: "pause" }) + const rejectedStart = assert.rejects(racedStart, /already running/) + await resumed + await rejectedStart + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId === "one").length, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps interpreter node references attached when pause persistence fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-rollback-")) + let releaseFirst!: () => void + let releaseSecond!: () => void + const firstBlocked = new Promise((resolve) => { releaseFirst = resolve }) + const secondBlocked = new Promise((resolve) => { releaseSecond = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `pause-rollback-${++sessions}` } }), + prompt: async () => { + prompts++ + await (prompts === 1 ? firstBlocked : secondBlocked) + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause-rollback", name: "Pause rollback", root: { + type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause-rollback" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + const persist = (manager as any).persist.bind(manager) + let pauseWriteStarted!: () => void + const pauseWrite = new Promise((resolve) => { pauseWriteStarted = resolve }) + let rejectPauseWrite!: () => void + const pauseWriteFailure = new Promise((_, reject) => { rejectPauseWrite = () => reject(new Error("pause write failed")) }) + let failed = false + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (!failed && run.status === "pausing") { + failed = true + pauseWriteStarted() + return pauseWriteFailure + } + return persist(run, touch) + } + const pausing = manager.pause(started.id) + await pauseWrite + releaseFirst() + while (prompts < 2 || started.usage?.tokens !== 10) await new Promise((resolve) => setTimeout(resolve, 1)) + rejectPauseWrite() + await assert.rejects(pausing, /pause write failed/) + assert.equal(started.status, "running") + assert.equal(started.pauseRequested, undefined) + assert.equal(started.usage?.tokens, 10) + assert.deepEqual(started.executionNodes?.filter((node) => node.type === "agent").map((node) => node.status), ["completed", "running"]) + releaseSecond() + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + assert.deepEqual(run.executionNodes?.filter((node) => node.type === "agent").map((node) => node.status), ["completed", "completed"]) + } finally { + releaseFirst() + releaseSecond() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps a parallel pause pausing until every worker reaches a boundary", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-parallel-pause-")) + const releases: Array<() => void> = [] + const blocked = [0, 1].map(() => new Promise((resolve) => releases.push(resolve))) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `parallel-pause-${++sessions}` } }), + prompt: async () => { + const index = prompts++ + if (index < 2) await blocked[index] + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "parallel-pause", name: "Parallel pause", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "sequence", id: "left", steps: [ + { type: "agent", id: "left-one", instructions: "One" }, + { type: "agent", id: "left-two", instructions: "Two" }, + ] }, + { type: "sequence", id: "right", steps: [ + { type: "agent", id: "right-one", instructions: "One" }, + { type: "agent", id: "right-two", instructions: "Two" }, + ] }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "parallel-pause" }) + while (prompts < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal((await manager.pause(started.id))?.status, "pausing") + releases[0]!() + while (!started.executionNodes?.some((node) => node.definitionNodeId.endsWith("-one") && node.status === "completed")) { + await new Promise((resolve) => setTimeout(resolve, 1)) + } + assert.equal(started.status, "pausing") + assert.equal(prompts, 2) + releases[1]!() + await waitFor(manager, started.id, ["paused"]) + assert.equal(prompts, 2) + await manager.resume(started.id) + await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 4) + } finally { + for (const release of releases) release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("suspends a limiter waiter before it creates a session", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pause-waiter-")) + let release!: () => void + const blocked = new Promise((resolve) => { release = resolve }) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `pause-waiter-${++sessions}` } }), + prompt: async () => { prompts++; if (prompts === 1) await blocked; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "pause-waiter", name: "Pause waiter", maxConcurrency: 2, + budget: { maxTokens: 100 }, root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "pause-waiter" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.pause(started.id) + release() + await waitFor(manager, started.id, ["paused"]) + assert.equal(prompts, 1) + assert.equal(sessions, 2) + await manager.resume(started.id) + await waitFor(manager, started.id, ["completed"]) + assert.equal(prompts, 2) + } finally { + release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("validates an input gate answer and stops after an observed budget overrun", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-budget-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(2, 100), parts: [{ type: "text", text: "costly" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gate", name: "Gate", budget: { maxCost: 1 }, root: { + type: "sequence", id: "root", steps: [ + { type: "gate", id: "input", gate: "input", prompt: "Name", inputSchema: { + type: "object", required: ["name"], properties: { name: { type: "string" } }, additionalProperties: false, + } }, + { type: "agent", id: "costly", instructions: "Spend" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_input"]) + const gateId = waiting.pendingGate!.executionNodeId + await assert.rejects(manager.answer(started.id, gateId, {}), /name is required/) + await manager.answer(started.id, gateId, { name: "Ada" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /cost budget/) + assert.equal(run.usage?.cost, 2) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fans cancellation out to every active parallel session", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-cancel-")) + let sessions = 0 + let prompts = 0 + const aborted = new Set() + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `session-${++sessions}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + return new Promise((_, reject) => options?.signal?.addEventListener("abort", () => reject(options.signal?.reason), { once: true })) + }, + abort: async ({ sessionID }: { sessionID: string }) => { aborted.add(sessionID); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "cancel", name: "Cancel", maxConcurrency: 2, root: { + type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "Wait" }, { type: "agent", id: "two", instructions: "Wait" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "cancel" }) + while (prompts < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.cancel(started.id) + const run = await waitFor(manager, started.id, ["cancelled"]) + assert.equal(aborted.size, 2) + assert.equal(run.executionNodes?.filter((node) => node.status === "cancelled").length, 3) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("aborts a child session created after cancellation begins", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-create-cancel-")) + let sessions = 0 + let releaseCreate!: () => void + const childCreate = new Promise((resolve) => { releaseCreate = resolve }) + const aborted: string[] = [] + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => { + const id = `late-${++sessions}` + if (sessions === 2) await childCreate + return { data: { id } } + }, + prompt: async () => { prompts++; return { data: { info: usage(), parts: [] } } }, + abort: async ({ sessionID }: { sessionID: string }) => { aborted.push(sessionID); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "create-cancel", name: "Create cancel", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "create-cancel" }) + while (sessions < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + const cancellation = manager.cancel(started.id) + releaseCreate() + assert.equal((await cancellation)?.status, "cancelled") + assert.deepEqual(aborted, ["late-2"]) + assert.equal(prompts, 0) + } finally { + releaseCreate() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("retains recovery ownership while cancelled session creation is unresolved", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-create-ambiguous-cancel-")) + let sessions = 0 + let releaseCreate!: () => void + const childCreate = new Promise((resolve) => { releaseCreate = resolve }) + const aborted: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => { + const id = `unresolved-${++sessions}` + if (sessions === 2) await childCreate + return { data: { id } } + }, + prompt: async () => ({ data: { info: usage(), parts: [] } }), + abort: async ({ sessionID }: { sessionID: string }) => { aborted.push(sessionID); return { data: false } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ + workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client, promptTimeoutMs: 25, + }) + try { + await manager.createDefinition({ version: 1, id: "create-ambiguous", name: "Create ambiguous", root: { + type: "agent", id: "work", instructions: "Wait", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "create-ambiguous" }) + while (sessions < 2) await new Promise((resolve) => setTimeout(resolve, 1)) + const cancelled = await manager.cancel(started.id) + assert.equal(cancelled?.status, "recovery_required") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "create-ambiguous" }), /already running/) + + releaseCreate() + const recovered = await waitFor(manager, started.id, ["recovery_required"]) + assert.equal(recovered.status, "recovery_required") + assert.ok(aborted.length > 0 && aborted.every((sessionId) => sessionId === "unresolved-2")) + } finally { + releaseCreate() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("does not retry an unconfirmed side effect and retains recovery reservation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-ambiguous-")) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `ambiguous-${++sessions}` } }), + prompt: async () => { prompts++; throw new Error("connection lost") }, + abort: async () => ({ data: false }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "ambiguous", name: "Ambiguous", root: { + type: "agent", id: "work", instructions: "Act", retry: { maxAttempts: 2, idempotent: true }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "ambiguous" }) + const run = await waitFor(manager, started.id, ["recovery_required"]) + assert.equal(prompts, 1) + assert.equal(run.executionNodes?.find((node) => node.definitionNodeId === "work")?.status, "interrupted") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "ambiguous" }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("resumes a confirmed-abort checkpoint captured during retry delay", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-retry-checkpoint-")) + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "retry-checkpoint", name: "Retry checkpoint", root: { + type: "agent", id: "work", instructions: "Retry", retry: { maxAttempts: 2, delayMs: 60_000, idempotent: true }, + } } + const run = { + id: "00000000-0000-4000-8000-000000000090", workspaceId: "workspace", workspaceLineageId: "lineage", + workspacePath: "C:/workspace", objective: "Retry safely", status: "running", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, executionNodes: [], + createdAt: now, updatedAt: now, + } as WorkflowRun + let prompts = 0 + let sessions = 0 + let checkpoint: WorkflowRun | undefined + const crash = new AbortController() + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `retry-checkpoint-${++sessions}` } }), + prompt: async () => { + if (++prompts === 1) throw new Error("retryable failure") + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const interpreter = new WorkflowInterpreter({ + run, client, + persist: async () => { + const action = run.executionNodes?.find((node) => node.definitionNodeId === "work") + if (!checkpoint && action?.status === "waiting" && action.attempt === 1 && !action.sessionIds?.length) { + checkpoint = JSON.parse(JSON.stringify(run)) as WorkflowRun + crash.abort(new Error("simulated crash")) + } + }, + signal: () => crash.signal, sessionStarted: () => true, sessionFinished: () => {}, + abortSession: async () => true, isCancelled: () => false, + }) + let manager: WorkflowManager | undefined + try { + await assert.rejects(interpreter.execute(), /simulated crash/) + assert.equal(checkpoint?.executionNodes?.[0]?.attempt, 1) + assert.equal(checkpoint?.executionNodes?.[0]?.status, "waiting") + assert.equal(checkpoint?.executionNodes?.[0]?.sessionIds, undefined) + await fs.writeFile(path.join(directory, `${run.id}.json`), JSON.stringify(checkpoint), "utf8") + manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + const recovered = (await manager.get(run.id))! + assert.equal(recovered.status, "interrupted") + assert.equal(recovered.executionNodes?.[0]?.attempt, 1) + await manager.resume(run.id) + const completed = await waitFor(manager, run.id, ["completed"]) + assert.equal(completed.executionNodes?.[0]?.attempt, 2) + assert.equal(prompts, 2) + } finally { + await manager?.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("cancels a confirmed retry checkpoint without requiring recovery", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-retry-cancel-")) + const now = new Date().toISOString() + const id = "00000000-0000-4000-8000-000000000091" + const definition: WorkflowDefinitionV1 = { version: 1, id: "retry-cancel", name: "Retry cancel", root: { + type: "agent", id: "work", instructions: "Retry", retry: { maxAttempts: 2, idempotent: true }, + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Cancel safe retry", status: "interrupted", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "waiting", attempt: 1, startedAt: now }], createdAt: now, updatedAt: now, + }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => null }) + try { + assert.equal((await manager.cancel(id))?.status, "cancelled") + assert.equal((await manager.get(id))?.status, "cancelled") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes budgeted actions and enforces usage before admitting another action", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-budget-admission-")) + let prompts = 0 + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `budget-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: usage(0.1, 11), parts: [{ type: "text", text: "spent" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "budget-admission", name: "Budget", maxConcurrency: 3, + budget: { maxTokens: 10 }, root: { type: "parallel", id: "root", maxConcurrency: 3, branches: [ + { type: "agent", id: "one", instructions: "One" }, + { type: "agent", id: "two", instructions: "Two" }, + { type: "agent", id: "three", instructions: "Three" }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "budget-admission" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /token budget 10/) + assert.equal(prompts, 1) + assert.equal(run.usage?.tokens, 11) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses one action deadline across tool lookup, session creation and retries", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-deadline-")) + const signals: AbortSignal[] = [] + let sessions = 0 + let prompts = 0 + const client = { + tool: { ids: async (_input: unknown, options?: { signal?: AbortSignal }) => { + signals.push(options!.signal!); return { data: ["read"] } + } }, + session: { + create: async (_input: unknown, options?: { signal?: AbortSignal }) => { + sessions++ + if (sessions > 1) signals.push(options!.signal!) + return { data: { id: `deadline-${sessions}` } } + }, + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + signals.push(options!.signal!) + return new Promise((_, reject) => options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true })) + }, + abort: async () => ({ data: true }), + }, + } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "deadline", name: "Deadline", root: { + type: "agent", id: "work", instructions: "Wait", timeoutMs: 500, retry: { maxAttempts: 2, idempotent: true }, + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "deadline" }) + const failed = await waitFor(manager, started.id, ["failed"]) + assert.equal(prompts, 1) + assert.equal(sessions, 2) + assert.ok(signals.every((signal) => signal === signals[0])) + assert.equal(failed.executionNodes?.find((node) => node.definitionNodeId === "work")?.attempt, 1) + assert.equal(failed.executionNodes?.find((node) => node.definitionNodeId === "work")?.sessionIds, undefined) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("requires explicit recovery before an ambiguous persisted side effect can repeat", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-")) + const id = "00000000-0000-4000-8000-000000000099" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { + version: 1, id: "recover", name: "Recover", + root: { type: "agent", id: "work", sessionKey: "worker", instructions: "Potential side effect" }, + } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Recover", status: "running", rootSessionId: "root", steps: [], revision: 2, + definitionId: "recover", definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ + id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", status: "running", + attempt: 1, sessionIds: ["ambiguous"], startedAt: now, + }], + sessionBindings: { worker: "ambiguous" }, + createdAt: now, updatedAt: now, + }), "utf8") + let sessions = 0 + const calls: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => { const id = `recovery-${++sessions}`; calls.push(`create:${id}`); return { data: { id } } }, + prompt: async () => { calls.push("prompt"); return { data: { info: usage(), parts: [{ type: "text", text: "confirmed" }] } } }, + abort: async ({ sessionID }: { sessionID: string }) => { calls.push(`abort:${sessionID}`); return { data: true } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const recovered = (await manager.get(id))! + assert.equal(recovered.status, "recovery_required") + assert.equal(recovered.executionNodes?.[0]?.status, "interrupted") + await assert.rejects(manager.resume(id), /Recovery confirmation is required/) + const recoveryRevision = recovered.revision! + await assert.rejects(manager.resume(id, true, recoveryRevision - 1), /confirmation is stale/) + await manager.resume(id, true, recoveryRevision) + const completed = await waitFor(manager, id, ["completed"]) + assert.equal(completed.executionNodes?.[0]?.output, "confirmed") + assert.deepEqual(calls.slice(0, 3), ["abort:ambiguous", "create:recovery-1", "prompt"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps recovery required when persisted session abort is unconfirmed", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-failed-")) + const id = "00000000-0000-4000-8000-000000000098" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { + version: 1, id: "recover-failed", name: "Recover failed", + root: { type: "agent", id: "work", instructions: "Potential side effect" }, + } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Recover", status: "recovery_required", rootSessionId: "root", steps: [], revision: 2, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "interrupted", attempt: 1, sessionIds: ["ambiguous"], startedAt: now, completedAt: now }], + createdAt: now, updatedAt: now, + }), "utf8") + let creates = 0 + const client = { tool: workflowTools, session: { + create: async () => { creates++; return { data: { id: "unexpected" } } }, + abort: async () => ({ data: false }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition(definition) + const recovered = (await manager.get(id))! + await assert.rejects(manager.resume(id, true, recovered.revision), /could not confirm/) + const run = (await manager.get(id))! + assert.equal(run.status, "recovery_required") + assert.equal(run.executionNodes?.[0]?.status, "interrupted") + assert.equal(creates, 0) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: definition.id }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("recovers session-bearing paused nodes independent of node type before answering or cancelling", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-paused-session-")) + const id = "00000000-0000-4000-8000-000000000095" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "paused-session", name: "Paused", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Approve", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Paused", status: "paused", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "gate", definitionNodeId: "gate", type: "gate", + status: "waiting", attempt: 0, sessionIds: ["unexpected-session"], startedAt: now }], + pendingGate: { executionNodeId: "execution", definitionNodeId: "gate", gate: "approval", prompt: "Approve" }, + createdAt: now, updatedAt: now, + }), "utf8") + let confirmAbort = false + const aborted: string[] = [] + const client = { session: { + abort: async ({ sessionID }: { sessionID: string }) => { aborted.push(sessionID); return { data: confirmAbort } }, + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const recovered = (await manager.get(id))! + assert.equal(recovered.status, "recovery_required") + assert.equal(recovered.executionNodes?.[0]?.status, "interrupted") + await assert.rejects(manager.answer(id, "execution", true), /not waiting for a gate answer/) + assert.equal((await manager.cancel(id))?.status, "recovery_required") + confirmAbort = true + assert.equal((await manager.cancel(id))?.status, "cancelled") + assert.deepEqual(aborted, ["unexpected-session", "unexpected-session"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes lineage admission and rejects a stale answer after the next gate opens", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-admission-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gates", name: "Gates", root: { + type: "sequence", id: "root", steps: [ + { type: "gate", id: "first", gate: "approval", prompt: "First" }, + { type: "gate", id: "second", gate: "approval", prompt: "Second" }, + ], + } }) + const starts = await Promise.allSettled([ + manager.start({ workspaceId: "workspace", definitionId: "gates" }), + manager.start({ workspaceId: "workspace", definitionId: "gates" }), + ]) + assert.equal(starts.filter((result) => result.status === "fulfilled").length, 1) + assert.match((starts.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /already running/) + const started = (starts.find((result) => result.status === "fulfilled") as PromiseFulfilledResult).value + const first = await waitFor(manager, started.id, ["waiting_for_review"]) + const firstGate = first.pendingGate!.executionNodeId + const answers = await Promise.allSettled([ + manager.answer(started.id, firstGate, true), + manager.answer(started.id, firstGate, true), + ]) + assert.equal(answers.filter((result) => result.status === "fulfilled").length, 1) + assert.match((answers.find((result) => result.status === "rejected") as PromiseRejectedResult).reason.message, /stale/) + const second = await waitFor(manager, started.id, ["waiting_for_review"]) + assert.equal(second.pendingGate?.definitionNodeId, "second") + await manager.approve(started.id, second.pendingGate!.executionNodeId) + await waitFor(manager, started.id, ["completed"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("pins and executes nested saved definitions with shared inputs, concurrency and budgets", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nested-")) + let sessions = 0 + let prompts = 0 + let active = 0 + let maxActive = 0 + let release!: () => void + const blocked = new Promise((resolve) => { release = resolve }) + const seenPrompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `nested-${++sessions}` } }), + prompt: async (input: Record) => { + prompts++ + active++ + maxActive = Math.max(maxActive, active) + seenPrompts.push(JSON.stringify(input.parts)) + if (prompts === 1) await blocked + active-- + return { data: { info: usage(0.1, 10), parts: [{ type: "text", text: "nested-output" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ + version: 1, id: "child", name: "Child", budget: { maxTokens: 25 }, maxConcurrency: 1, + root: { type: "parallel", id: "child-root", branches: [ + { type: "agent", id: "child-work-1", instructions: "Old child instructions", context: { $ref: "inputs.message" } }, + { type: "agent", id: "child-work-2", instructions: "Old child instructions", context: { $ref: "inputs.message" } }, + ] }, + }) + await manager.createDefinition({ + version: 1, id: "parent", name: "Parent", maxConcurrency: 2, budget: { maxTokens: 15 }, + root: { type: "workflow", id: "nested", definitionId: "child", inputs: { message: { $ref: "inputs.message" } } }, + }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "parent", inputs: { message: "hello" } }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await manager.updateDefinition("child", 1, { + version: 1, id: "child", name: "Child", + root: { type: "agent", id: "child-work", instructions: "New child instructions" }, + }) + release() + const run = await waitFor(manager, started.id, ["failed"]) + assert.equal(run.savedDefinitionSnapshots?.length, 1) + assert.equal(run.savedDefinitionSnapshots?.[0]?.revision, 1) + assert.equal((run.definitionSnapshot?.root as { definitionRevision?: number }).definitionRevision, 1) + assert.equal(run.executionNodes?.filter((node) => node.definitionNodeId.startsWith("child-work-")).length, 2) + assert.equal(run.usage?.tokens, 20) + assert.match(run.error ?? "", /cost budget|token budget 15/) + assert.equal(maxActive, 1) + assert.match(seenPrompts.join("\n"), /Old child instructions/) + assert.match(seenPrompts.join("\n"), /hello/) + assert.doesNotMatch(seenPrompts.join("\n"), /New child instructions/) + } finally { + release() + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("shares a saved-definition budget admission lock across parallel invocations", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-shared-nested-budget-")) + let sessions = 0 + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `shared-budget-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: usage(0, 11), parts: [{ type: "text", text: "spent" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "shared-budget-child", name: "Child", budget: { maxTokens: 10 }, root: { + type: "agent", id: "spend", instructions: "Spend", + } }) + await manager.createDefinition({ version: 1, id: "shared-budget-parent", name: "Parent", maxConcurrency: 2, root: { + type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "workflow", id: "first", definitionId: "shared-budget-child" }, + { type: "workflow", id: "second", definitionId: "shared-budget-child" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "shared-budget-parent" }) + assert.match((await waitFor(manager, started.id, ["failed"])).error ?? "", /token budget 10/) + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("keeps nodes references inside one saved-definition invocation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-reference-scope-")) + let sessions = 0 + const prompts: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `scope-${++sessions}` } }), + prompt: async (input: Record) => { + const prompt = JSON.stringify(input.parts) + prompts.push(prompt) + const text = prompt.includes("Parent seed") ? "parent-value" : prompt.includes("Child seed") ? "child-value" : "done" + return { data: { info: usage(), parts: [{ type: "text", text }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "scope-child", name: "Child", root: { + type: "sequence", id: "child-root", steps: [ + { type: "agent", id: "seed", instructions: "Child seed" }, + { type: "agent", id: "consume", instructions: "Child consume", context: { $ref: "nodes.seed.output" } }, + ], + } }) + await manager.createDefinition({ version: 1, id: "scope-parent", name: "Parent", root: { + type: "sequence", id: "parent-root", steps: [ + { type: "agent", id: "seed", instructions: "Parent seed" }, + { type: "workflow", id: "child", definitionId: "scope-child" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "scope-parent" }) + await waitFor(manager, started.id, ["completed"]) + const consume = prompts.find((prompt) => prompt.includes("Child consume")) ?? "" + assert.match(consume, /child-value/) + assert.doesNotMatch(consume, /parent-value/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("defaults each nested workflow invocation to one concurrent action", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nested-default-")) + let sessions = 0 + let active = 0 + let maxActive = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `nested-default-${++sessions}` } }), + prompt: async () => { + active++ + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 5)) + active-- + return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "nested-child", name: "Child", root: { + type: "parallel", id: "child-root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + await manager.createDefinition({ version: 1, id: "nested-parent", name: "Parent", maxConcurrency: 2, root: { + type: "workflow", id: "child", definitionId: "nested-child", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "nested-parent" }) + await waitFor(manager, started.id, ["completed"]) + assert.equal(maxActive, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("cancels nested limiter waiters without deadlocking an invalid transition or shutdown", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-limiter-cancel-")) + let sessions = 0 + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `limiter-${++sessions}` } }), + prompt: async (_input: unknown, options?: { signal?: AbortSignal }) => { + prompts++ + return new Promise((_, reject) => options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true })) + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "limiter-child", name: "Child", root: { + type: "parallel", id: "child-root", maxConcurrency: 2, branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + await manager.createDefinition({ version: 1, id: "limiter-parent", name: "Parent", maxConcurrency: 2, root: { + type: "workflow", id: "child", definitionId: "limiter-child", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "limiter-parent" }) + while (prompts === 0) await new Promise((resolve) => setTimeout(resolve, 1)) + await assert.rejects(Promise.race([ + manager.resume(started.id), + new Promise((_, reject) => setTimeout(() => reject(new Error("resume deadlocked")), 100)), + ]), /cannot be resumed/) + await manager.shutdown() + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("serializes current-workspace rebinding against authoritative persisted state", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-bind-")) + const id = "00000000-0000-4000-8000-000000000097" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "bound", name: "Bound", root: { + type: "agent", id: "work", instructions: "Done", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "old", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Bound", status: "interrupted", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + worktreeSelection: { policy: { mode: "current" }, sourceWorkspaceId: "old", sourceWorkspaceLineageId: "lineage", + sourceWorkspacePath: "C:/workspace", workspaceId: "old", directory: "C:/workspace", created: false }, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "completed", attempt: 1, output: "done", startedAt: now, completedAt: now }], + createdAt: now, updatedAt: now, + }), "utf8") + const restoredWorkspaces = { + get: (workspaceId: string) => workspaceId === "restored" + ? { id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" } + : undefined, + list: () => [{ id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ workspaceManager: restoredWorkspaces, eventBus, logger, storageDir: directory }) + try { + const persist = (manager as any).persist.bind(manager) + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (run.workspaceId === "restored") throw new Error("bind write failed") + return persist(run, touch) + } + await assert.rejects(manager.get(id, "restored"), /bind write failed/) + assert.equal((await manager.get(id))?.workspaceId, "old") + assert.equal((manager as any).activeWorkspaces.get("old"), id) + assert.equal((manager as any).activeWorkspaces.has("restored"), false) + ;(manager as any).persist = persist + const [run, listed] = await Promise.all([manager.get(id, "restored"), manager.list("restored")]) + assert.equal(run?.workspaceId, "restored") + assert.equal(run?.worktreeSelection?.workspaceId, "restored") + assert.equal(run?.worktreeSelection?.sourceWorkspaceId, "restored") + assert.equal(listed[0]?.workspaceId, "restored") + const stored = JSON.parse(await fs.readFile(path.join(directory, `${id}.json`), "utf8")) as WorkflowRun + assert.equal(stored.worktreeSelection?.sourceWorkspaceId, "restored") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("skips corrupt history and never prunes interrupted runs", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-history-")) + const now = new Date().toISOString() + for (let index = 0; index < 101; index++) await fs.writeFile(path.join(directory, `complete-${index}.json`), JSON.stringify({ + id: `complete-${index}`, workspaceId: `history-${index % 2}`, workspaceLineageId: `history-lineage-${index % 2}`, + workspacePath: `C:/history-${index % 2}`, + objective: "History", status: "completed", steps: [], createdAt: now, updatedAt: new Date(Date.now() + index).toISOString(), + }), "utf8") + await fs.writeFile(path.join(directory, "interrupted.json"), JSON.stringify({ + id: "interrupted", workspaceId: "interrupted-workspace", workspaceLineageId: "interrupted-lineage", workspacePath: "C:/interrupted", + objective: "Resume me", status: "interrupted", steps: [], createdAt: "9999-01-01T00:00:00.000Z", updatedAt: now, + }), "utf8") + await fs.writeFile(path.join(directory, "corrupt.json"), JSON.stringify({ id: "corrupt", status: "completed" }), "utf8") + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `history-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + assert.equal((await manager.list()).some((run) => run.id === "interrupted"), true) + const started = await manager.start({ workspaceId: "workspace", objective: "Prune", stages: [{ id: "stage", title: "Stage", instructions: "Run" }] }) + await waitFor(manager, started.id, ["completed"]) + const entries = await fs.readdir(directory) + assert.equal(entries.includes("interrupted.json"), true) + assert.equal(entries.includes("corrupt.json"), true) + assert.equal(entries.filter((entry) => entry.endsWith(".json") && entry !== "interrupted.json" && entry !== "corrupt.json").length, 100) + await assert.doesNotReject(manager.list()) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("rejects saved workflow cycles and excessive nesting before creating sessions", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-nesting-")) + let sessions = 0 + const client = { tool: workflowTools, session: { create: async () => { sessions++; return { data: { id: "unexpected" } } } } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "cycle-a", name: "A", root: { type: "workflow", id: "call-b", definitionId: "cycle-b" } }) + await manager.createDefinition({ version: 1, id: "cycle-b", name: "B", root: { type: "workflow", id: "call-a", definitionId: "cycle-a" } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "cycle-a" }), /cycle-a -> cycle-b -> cycle-a/) + + for (let index = 9; index >= 0; index -= 1) await manager.createDefinition({ + version: 1, id: `depth-${index}`, name: `Depth ${index}`, + root: index === 9 + ? { type: "condition", id: "leaf", condition: false, then: { type: "agent", id: "never", instructions: "Never" } } + : { type: "workflow", id: `call-${index + 1}`, definitionId: `depth-${index + 1}` }, + }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "depth-0" }), /maximum depth 8/) + assert.equal(sessions, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("launches new and existing managed worktrees as retained OpenCode workspaces", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-worktree-run-")) + const repo = path.join(directory, "repo") + await fs.mkdir(repo) + await execFileAsync("git", ["init"], { cwd: repo }) + await fs.writeFile(path.join(repo, "README.md"), "initial\n") + await execFileAsync("git", ["add", "README.md"], { cwd: repo }) + await execFileAsync("git", ["-c", "user.name=CodeNomad", "-c", "user.email=test@example.com", "commit", "-m", "initial"], { cwd: repo }) + + const descriptors = new Map([["source", { + id: "source", lineageId: "source-lineage", path: repo, status: "ready", binaryId: "opencode", + }]]) + const launchedDirectories: string[] = [] + const cancelledCreationRequests: string[] = [] + let workspaceNumber = 0 + let retainCreation = true + let manager!: WorkflowManager + const managedWorkspaces = { + get: (id: string) => descriptors.get(id), + list: () => Array.from(descriptors.values()), + create: async (folder: string) => { + launchedDirectories.push(folder) + const existing = Array.from(descriptors.values()).find((entry) => path.resolve(entry.path) === path.resolve(folder)) + if (existing) return { workspace: existing, created: false } + const id = `target-${++workspaceNumber}` + const workspace = { id, lineageId: `${id}-lineage`, path: folder, status: "ready", binaryId: "opencode" } + descriptors.set(id, workspace) + return { workspace, created: true } + }, + releaseCreationRequest: () => retainCreation, + cancelCreationRequest: async (requestId: string) => { + cancelledCreationRequests.push(requestId) + assert.equal(await manager.withWorkspaceOwnershipLease({ id: `target-${workspaceNumber}` }, async (owned) => owned), false) + }, + } as unknown as WorkspaceManager + const clientWorkspaceIds: string[] = [] + const publishedWorkspaceIds: string[] = [] + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `worktree-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + manager = new WorkflowManager({ + workspaceManager: managedWorkspaces, + eventBus: { publish: (event: { instanceId: string }) => { publishedWorkspaceIds.push(event.instanceId); return true } } as unknown as EventBus, + logger, + storageDir: path.join(directory, "runs"), definitionsDir: path.join(directory, "definitions"), + createClient: (workspaceId) => { clientWorkspaceIds.push(workspaceId); return client }, + }) + const prunedLineages: string[] = [] + const pruneHistory = (manager as any).pruneHistory.bind(manager) + ;(manager as any).pruneHistory = async (lineageId: string) => { + prunedLineages.push(lineageId) + return pruneHistory(lineageId) + } + try { + await manager.createDefinition({ version: 1, id: "isolated", name: "Isolated", root: { type: "agent", id: "work", instructions: "Work" } }) + await assert.rejects(manager.start({ + workspaceId: "source", definitionId: "isolated", initiatorSessionId: "source-session", + worktree: { mode: "new", slug: "workflow-test" }, + }), /initiatorSessionId is unsupported/) + const created = await manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { mode: "new", slug: "workflow-test" } }) + const first = await waitFor(manager, created.id, ["completed"]) + assert.equal(first.worktreeSelection?.created, true) + assert.equal(first.worktreeSelection?.slug, "workflow-test") + assert.equal(first.workspacePath, launchedDirectories[0]) + assert.equal(first.workspaceId, clientWorkspaceIds[0]) + assert.ok(publishedWorkspaceIds.includes("source")) + assert.ok(publishedWorkspaceIds.includes(first.workspaceId)) + await fs.writeFile(path.join(first.workspacePath, "dirty.txt"), "retain me\n") + + const reused = await manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { mode: "existing", slug: "workflow-test" } }) + const second = await waitFor(manager, reused.id, ["completed"]) + assert.equal(second.worktreeSelection?.created, false) + assert.equal(second.workspacePath, first.workspacePath) + assert.deepEqual(prunedLineages, ["source-lineage", "source-lineage"]) + assert.equal(await fs.readFile(path.join(first.workspacePath, "dirty.txt"), "utf8"), "retain me\n") + retainCreation = false + await assert.rejects(manager.start({ workspaceId: "source", definitionId: "isolated", worktree: { + mode: "existing", slug: "workflow-test", + } }), /ownership could not be retained/) + assert.equal(cancelledCreationRequests.length, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("resumes an action that crashed before admission without recovery confirmation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pre-admission-recovery-")) + const id = "00000000-0000-4000-8000-000000000093" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "pre-admission", name: "Pre-admission", root: { + type: "agent", id: "work", instructions: "Run once admitted", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Resume safely", status: "running", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "running", attempt: 0, startedAt: now }], createdAt: now, updatedAt: now, + }), "utf8") + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: "admitted" } }), + prompt: async () => { prompts++; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const recovered = (await manager.get(id))! + assert.equal(recovered.status, "interrupted") + assert.equal(recovered.executionNodes?.[0]?.status, "waiting") + await manager.resume(id) + await waitFor(manager, id, ["completed"]) + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("restores completed repeat output when its parent checkpoint was interrupted", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-repeat-recovery-")) + const id = "00000000-0000-4000-8000-000000000092" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "repeat-recovery", name: "Repeat recovery", root: { + type: "repeat", id: "retry", maxIterations: 1, + while: { value: { $ref: "nodes.work.output" }, notEquals: "done" }, onExhausted: "fail", + body: { type: "agent", id: "work", instructions: "Work" }, + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Recover repeat", status: "running", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [ + { id: "repeat", instanceKey: "retry", definitionNodeId: "retry", type: "repeat", status: "running", attempt: 0, startedAt: now }, + { id: "work", instanceKey: "retry/work[0]", parentInstanceKey: "retry", definitionNodeId: "work", type: "agent", + status: "completed", attempt: 1, output: "done", startedAt: now, completedAt: now }, + ], createdAt: now, updatedAt: now, + }), "utf8") + const client = { tool: workflowTools, session: { + create: async () => { throw new Error("completed repeat body must not run again") }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + assert.equal((await manager.get(id))?.status, "interrupted") + await manager.resume(id) + const completed = await waitFor(manager, id, ["completed"]) + assert.deepEqual(completed.executionNodes?.find((node) => node.id === "repeat")?.output, ["done"]) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("never repeats crash-interrupted actions without persisted termination evidence", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-no-recovery-evidence-")) + const id = "00000000-0000-4000-8000-000000000096" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "no-evidence", name: "No evidence", root: { + type: "agent", id: "work", instructions: "Do not repeat", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "No repeat", status: "running", rootSessionId: "root", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "running", attempt: 1, startedAt: now }], createdAt: now, updatedAt: now, + }), "utf8") + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: "unexpected" } }), + prompt: async () => { prompts++; return { data: { info: usage(), parts: [] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + assert.equal((await manager.get(id))?.status, "recovery_required") + const recovered = (await manager.get(id))! + await assert.rejects(manager.resume(id, true, recovered.revision), /no persisted session IDs.*will not be repeated/) + assert.equal((await manager.get(id))?.status, "recovery_required") + assert.equal(prompts, 0) + await assert.rejects(manager.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("restores durable reservation state when cancel and resume persistence fail", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-transition-rollback-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `rollback-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "rollback", name: "Rollback", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Wait", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "rollback" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_review"]) + const persist = (manager as any).persist.bind(manager) + let failCancel = true + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (failCancel && run.status === "cancelled") { failCancel = false; throw new Error("cancel write failed") } + return persist(run, touch) + } + await assert.rejects(manager.cancel(started.id), /cancel write failed/) + assert.equal((await manager.get(started.id))?.status, "waiting_for_review") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "rollback" }), /already running/) + + let failResume = true + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (failResume && run.status === "running") { failResume = false; throw new Error("resume write failed") } + return persist(run, touch) + } + await assert.rejects(manager.answer(started.id, waiting.pendingGate!.executionNodeId, true), /resume write failed/) + const restored = await manager.get(started.id) + assert.equal(restored?.status, "waiting_for_review") + assert.equal(restored?.pendingGate?.executionNodeId, waiting.pendingGate!.executionNodeId) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "rollback" }), /already running/) + ;(manager as any).persist = persist + assert.equal((await manager.cancel(started.id))?.status, "cancelled") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("blocks at exact observed budget and rejects malformed SDK usage", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-strict-budget-")) + let prompts = 0 + let sessions = 0 + let invalid = false + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `strict-${++sessions}` } }), + prompt: async () => { + prompts++ + return { data: { info: invalid ? usage(-1, Number.NaN) : usage(0, 10), parts: [{ type: "text", text: "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "exact-budget", name: "Exact", budget: { maxTokens: 10 }, root: { + type: "sequence", id: "root", steps: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const exact = await manager.start({ workspaceId: "workspace", definitionId: "exact-budget" }) + const exactRun = await waitFor(manager, exact.id, ["failed"]) + assert.match(exactRun.error ?? "", /reached token budget 10/) + assert.equal(prompts, 1) + + invalid = true + await manager.createDefinition({ version: 1, id: "invalid-usage", name: "Invalid usage", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const malformed = await manager.start({ workspaceId: "workspace", definitionId: "invalid-usage" }) + const malformedRun = await waitFor(manager, malformed.id, ["failed"]) + assert.match(malformedRun.error ?? "", /usage .* must be finite and non-negative/) + assert.equal(malformedRun.usage?.cost, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fails usage addition before a non-finite value can be persisted", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-usage-overflow-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `overflow-${++sessions}` } }), + prompt: async () => ({ data: { + info: { role: "assistant", cost: 0, tokens: { + input: Number.MAX_VALUE, output: Number.MAX_VALUE, reasoning: 0, cache: { read: 0, write: 0 }, + } }, + parts: [{ type: "text", text: "done" }], + } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "usage-overflow", name: "Usage overflow", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "usage-overflow" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.match(run.error ?? "", /usage tokens.total overflowed/) + assert.equal(run.usage?.tokens, 0) + assert.doesNotMatch(await fs.readFile(path.join(directory, `${started.id}.json`), "utf8"), /"tokens": null/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("validates provider structured output and bounds resolved action context", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-output-context-")) + let prompts = 0 + let sessions = 0 + const seen: string[] = [] + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `validation-${++sessions}` } }), + prompt: async (input: Record) => { + prompts++ + seen.push(JSON.stringify(input.parts)) + return { data: { info: { ...usage(), structured: { count: "wrong" } }, parts: [] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "schema-check", name: "Schema", root: { + type: "agent", id: "work", instructions: "Work", outputSchema: { + type: "object", required: ["count"], properties: { count: { type: "number" } }, + }, + } }) + const structured = await manager.start({ workspaceId: "workspace", definitionId: "schema-check" }) + assert.match((await waitFor(manager, structured.id, ["failed"])).error ?? "", /Structured output is invalid/) + + await manager.createDefinition({ version: 1, id: "context-check", name: "Context", root: { + type: "agent", id: "work", instructions: "Work", context: { $ref: "inputs.payload" }, + } }) + const oversized = await manager.start({ workspaceId: "workspace", definitionId: "context-check", inputs: { payload: "x".repeat(256_001) } }) + assert.match((await waitFor(manager, oversized.id, ["failed"])).error ?? "", /context exceeds 256000 bytes/) + await manager.createDefinition({ version: 1, id: "own-ref", name: "Own ref", root: { + type: "agent", id: "work", instructions: "Work", context: { $ref: "inputs.toString" }, + } }) + const inherited = await manager.start({ workspaceId: "workspace", definitionId: "own-ref", inputs: {} }) + await waitFor(manager, inherited.id, ["completed"]) + assert.equal(prompts, 2) + assert.doesNotMatch(seen.at(-1) ?? "", /Context:/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("allows bounded aggregate structural output above the action leaf limit", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-structural-output-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `aggregate-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "x".repeat(9_000) }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "aggregate-output", name: "Aggregate", maxConcurrency: 2, root: { + type: "parallel", id: "root", branches: [ + { type: "agent", id: "one", instructions: "One" }, { type: "agent", id: "two", instructions: "Two" }, + ], + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "aggregate-output" }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.ok(JSON.stringify(run.executionNodes?.find((node) => node.definitionNodeId === "root")?.output).length > 16_000) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("rejects composed expansion and aggregate saved graph bytes before effects", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-composed-limits-")) + let sessions = 0 + const client = { session: { create: async () => { sessions++; return { data: { id: "unexpected" } } } } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "expansion-child", name: "Child", root: { + type: "foreach", id: "each", items: [], item: "item", maxItems: 100, + body: { type: "agent", id: "work", instructions: "Work" }, + } }) + await manager.createDefinition({ version: 1, id: "expansion-parent", name: "Parent", root: { + type: "foreach", id: "outer", items: [], item: "item", maxItems: 100, + body: { type: "workflow", id: "child", definitionId: "expansion-child" }, + } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "expansion-parent" }), /expand above limit/) + + const byteIds = ["bytes-a", "bytes-b", "bytes-c", "bytes-d", "bytes-e", "bytes-f"] + for (const id of byteIds) await manager.createDefinition({ version: 1, id, name: id, root: { + type: "agent", id: "work", instructions: "x".repeat(45_000), + } }) + await manager.createDefinition({ version: 1, id: "bytes-parent", name: "Bytes", root: { + type: "sequence", id: "root", steps: byteIds.map((id, index) => ({ + type: "workflow" as const, id: `call-${index}`, definitionId: id, + })), + } }) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "bytes-parent" }), /graph exceeds 256000 bytes/) + assert.equal(sessions, 0) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("fails admission closed for malformed active records and quarantines identifiable ownership", async () => { + const blockedDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-malformed-global-")) + await fs.writeFile(path.join(blockedDir, "unknown.json"), "{not json", "utf8") + const blocked = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: blockedDir }) + try { + await assert.rejects(blocked.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /malformed active run unknown.json/) + assert.equal(await blocked.isWorkspaceWorkflowOwned({ lineageId: "anything" }), true) + } finally { + await blocked.shutdown() + await fs.rm(blockedDir, { recursive: true, force: true }) + } + + const quarantinedDir = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-malformed-lineage-")) + await fs.writeFile(path.join(quarantinedDir, "known.json"), JSON.stringify({ + id: "known", status: "running", workspaceId: "old", workspaceLineageId: "quarantined", workspacePath: "C:/quarantined", + }), "utf8") + const quarantined = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: quarantinedDir }) + try { + assert.equal(await quarantined.isWorkspaceWorkflowOwned({ lineageId: "quarantined" }), true) + assert.equal(await quarantined.isWorkspaceWorkflowOwned({ path: "C:/other" }), false) + } finally { + await quarantined.shutdown() + await fs.rm(quarantinedDir, { recursive: true, force: true }) + } + }) + + it("rejects persisted execution types that disagree with the pinned graph", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-pinned-node-type-")) + const id = "wrong-type" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "typed", name: "Typed", root: { + type: "agent", id: "work", instructions: "Work", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "typed-lineage", workspacePath: "C:/typed", + objective: "Typed", status: "paused", steps: [], revision: 1, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "sequence", + status: "waiting", attempt: 0 }], createdAt: now, updatedAt: now, + }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + await assert.rejects(manager.get(id), /pinned graph/) + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "typed-lineage" }), true) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("exposes uncapped canonical ownership leases for retained execution worktrees", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-ownership-")) + const now = new Date().toISOString() + for (let index = 0; index < 101; index++) await fs.writeFile(path.join(directory, `done-${index}.json`), JSON.stringify({ + id: `done-${index}`, workspaceId: "old", workspaceLineageId: "old", workspacePath: "C:/old", + objective: "Done", status: "completed", steps: [], createdAt: now, updatedAt: new Date(Date.now() + index).toISOString(), + }), "utf8") + await fs.writeFile(path.join(directory, "owned.json"), JSON.stringify({ + id: "owned", workspaceId: "execution", workspaceLineageId: "execution-lineage", workspacePath: "C:/repo/.worktrees/job", + objective: "Owned", status: "interrupted", steps: [], + worktreeSelection: { policy: { mode: "existing", slug: "job" }, sourceWorkspaceId: "source-old", + sourceWorkspaceLineageId: "source-lineage", sourceWorkspacePath: "C:/repo", workspaceId: "execution", + directory: "C:/repo/.worktrees/job", slug: "job", created: false }, + createdAt: "2000-01-01T00:00:00.000Z", updatedAt: now, + }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "source-lineage", path: "C:/repo" }), true) + assert.equal(await manager.isWorkspaceWorkflowOwned({ lineageId: "execution-lineage", path: "C:/repo/.worktrees/job" }), true) + assert.equal(await manager.isWorktreeWorkflowOwned({ lineageId: "source-lineage" }, { slug: "job" }), true) + assert.equal(await manager.isWorktreeWorkflowOwned({ lineageId: "wrong-source" }, { path: "C:/repo/.worktrees/job" }), true) + const leased = await manager.withWorktreeOwnershipLease( + { path: "C:/repo" }, { path: "C:/repo/.worktrees/job" }, async (owned) => owned, + ) + assert.equal(leased, true) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("prepares gate execution before consuming it and supports atomic owned cancellation", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-prepare-")) + let ready = true + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-prepare-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const ownedWorkspaces = { + get: (id: string) => id === "wrong-workspace" + ? { id, lineageId: "wrong-lineage", path: "C:/wrong", status: "ready" } + : { id, lineageId: "lineage", path: "C:/workspace", status: "ready" }, + list: () => [{ id: "restored", lineageId: "lineage", path: "C:/workspace", status: "ready" }], + } as unknown as WorkspaceManager + const manager = new WorkflowManager({ workspaceManager: ownedWorkspaces, eventBus, logger, storageDir: directory, createClient: () => ready ? client : null }) + try { + await manager.createDefinition({ version: 1, id: "gate-prepare", name: "Gate", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Approve", + } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate-prepare" }) + const waiting = await waitFor(manager, started.id, ["waiting_for_review"]) + ready = false + await assert.rejects(manager.answer(started.id, waiting.pendingGate!.executionNodeId, true), /not ready/) + assert.equal((await manager.get(started.id))?.pendingGate?.executionNodeId, waiting.pendingGate!.executionNodeId) + assert.equal(await manager.cancelOwned(started.id, "wrong-workspace"), undefined) + const cancelled = await manager.cancelOwned(started.id, "restored") + assert.equal(cancelled?.status, "cancelled") + assert.equal(cancelled?.workspaceId, "restored") + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses the persisted bounded legacy output for the next-stage handoff", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-legacy-handoff-")) + const tail = "UNBOUNDED-TAIL" + const prompts: string[] = [] + let sessions = 0 + const client = { session: { + create: async () => ({ data: { id: `legacy-handoff-${++sessions}` } }), + prompt: async (input: Record) => { + prompts.push(JSON.stringify(input.parts)) + return { data: { info: {}, parts: [{ type: "text", text: prompts.length === 1 ? `${"x".repeat(20_000)}${tail}` : "done" }] } } + }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + const started = await manager.start({ workspaceId: "workspace", objective: "Bound handoff", stages: [ + { id: "one", title: "One", instructions: "One" }, + { id: "two", title: "Two", instructions: "Two" }, + ] }) + const run = await waitFor(manager, started.id, ["completed"]) + assert.equal((run.steps[0]?.output as string).length, 16_000) + assert.equal(run.steps[0]?.outputTruncated, true) + assert.doesNotMatch(prompts[1] ?? "", new RegExp(tail)) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("clears a pending gate when a parallel branch fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-gate-branch-failure-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `gate-failure-${++sessions}` } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "gate-branch-failure", name: "Gate branch failure", maxConcurrency: 2, + root: { type: "parallel", id: "root", maxConcurrency: 2, branches: [ + { type: "gate", id: "gate", gate: "approval", prompt: "Wait" }, + { type: "agent", id: "fail", instructions: "Fail", tools: ["bash"] }, + ] } }) + const started = await manager.start({ workspaceId: "workspace", definitionId: "gate-branch-failure" }) + const run = await waitFor(manager, started.id, ["failed"]) + assert.equal(run.pendingGate, undefined) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("retains recovery ownership when a completed action checkpoint fails", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-action-checkpoint-")) + let sessions = 0 + let prompts = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `action-checkpoint-${++sessions}` } }), + prompt: async () => { prompts++; return { data: { info: usage(), parts: [{ type: "text", text: "done" }] } } }, + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + try { + await manager.createDefinition({ version: 1, id: "action-checkpoint", name: "Action checkpoint", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const persist = (manager as any).persist.bind(manager) + let failCheckpoint = true + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + const action = run.executionNodes?.find((node) => node.definitionNodeId === "work") + if (failCheckpoint && run.status === "running" && action?.status === "completed") { + failCheckpoint = false + throw new Error("action checkpoint failed") + } + return persist(run, touch) + } + const started = await manager.start({ workspaceId: "workspace", definitionId: "action-checkpoint" }) + const recovery = await waitFor(manager, started.id, ["recovery_required"]) + assert.equal(recovery.executionNodes?.find((node) => node.definitionNodeId === "work")?.status, "completed") + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "action-checkpoint" }), /already running/) + assert.equal((await manager.cancel(started.id))?.status, "cancelled") + assert.equal(prompts, 1) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("retains the reservation when terminal and fallback persistence both fail", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-terminal-persist-")) + let sessions = 0 + const client = { tool: workflowTools, session: { + create: async () => ({ data: { id: `terminal-persist-${++sessions}` } }), + prompt: async () => ({ data: { info: usage(), parts: [{ type: "text", text: "done" }] } }), + abort: async () => ({ data: true }), + } } as unknown as OpencodeClient + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + let reloaded: WorkflowManager | undefined + try { + await manager.createDefinition({ version: 1, id: "terminal-persist", name: "Terminal persist", root: { + type: "agent", id: "work", instructions: "Work", + } }) + const persist = (manager as any).persist.bind(manager) + ;(manager as any).persist = async (run: WorkflowRun, touch?: boolean) => { + if (run.status === "completed" || run.status === "failed" || run.status === "recovery_required") throw new Error("terminal write failed") + return persist(run, touch) + } + const started = await manager.start({ workspaceId: "workspace", definitionId: "terminal-persist" }) + for (let attempt = 0; attempt < 200 && !(manager as any).activeRuns.get(started.id)?.releaseBlocked; attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + assert.equal((manager as any).activeRuns.get(started.id)?.releaseBlocked, true) + const recovery = await manager.get(started.id) + assert.equal(recovery?.status, "recovery_required") + assert.equal(recovery?.pendingGate, undefined) + assert.equal(await fs.stat(path.join(directory, `${started.id}.recovery`)).then(() => true), true) + await assert.rejects(manager.start({ workspaceId: "workspace", definitionId: "terminal-persist" }), /already running/) + ;(manager as any).persist = persist + reloaded = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory, createClient: () => client }) + assert.equal((await reloaded.get(started.id))?.status, "recovery_required") + await assert.rejects(reloaded.start({ workspaceId: "workspace", definitionId: "terminal-persist" }), /already running/) + assert.equal((await reloaded.cancel(started.id))?.status, "cancelled") + const replacement = await reloaded.start({ workspaceId: "workspace", definitionId: "terminal-persist" }) + await waitFor(reloaded, replacement.id, ["completed"]) + } finally { + await manager.shutdown() + await reloaded?.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("uses a recovery marker instead of admitting an older waiting snapshot", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-recovery-marker-")) + const id = "00000000-0000-4000-8000-000000000094" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "marked-gate", name: "Marked gate", root: { + type: "gate", id: "gate", gate: "approval", prompt: "Do not approve", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Marked", status: "waiting_for_review", steps: [], revision: 2, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "gate", definitionNodeId: "gate", type: "gate", + status: "waiting", attempt: 0, startedAt: now }], + pendingGate: { executionNodeId: "execution", definitionNodeId: "gate", gate: "approval", prompt: "Do not approve" }, + createdAt: now, updatedAt: now, + }), "utf8") + await fs.writeFile(path.join(directory, `${id}.recovery`), JSON.stringify({ runId: id }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + const run = await manager.get(id) + assert.equal(run?.status, "recovery_required") + assert.equal(run?.pendingGate, undefined) + await assert.rejects(manager.approve(id, "execution"), /not waiting for approval/) + await assert.rejects(manager.start({ workspaceId: "workspace", objective: "blocked", stages: [] }), /already running/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + it("ignores a recovery marker superseded by a newer durable run revision", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "codenomad-stale-recovery-marker-")) + const id = "00000000-0000-4000-8000-000000000091" + const now = new Date().toISOString() + const definition: WorkflowDefinitionV1 = { version: 1, id: "resolved-marker", name: "Resolved marker", root: { + type: "agent", id: "work", instructions: "Done", + } } + await fs.writeFile(path.join(directory, `${id}.json`), JSON.stringify({ + id, workspaceId: "workspace", workspaceLineageId: "lineage", workspacePath: "C:/workspace", + objective: "Resolved", status: "completed", steps: [], revision: 5, + definitionId: definition.id, definitionRevision: 1, definitionSnapshot: definition, inputs: {}, + usage: { cost: 0, tokens: 0, inputTokens: 0, outputTokens: 0, reasoningTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, + executionNodes: [{ id: "execution", instanceKey: "work", definitionNodeId: "work", type: "agent", + status: "completed", attempt: 1, output: "done", startedAt: now, completedAt: now }], + createdAt: now, updatedAt: now, + }), "utf8") + await fs.writeFile(path.join(directory, `${id}.recovery`), JSON.stringify({ runId: id, revision: 4 }), "utf8") + const manager = new WorkflowManager({ workspaceManager, eventBus, logger, storageDir: directory }) + try { + assert.equal((await manager.get(id))?.status, "completed") + await assert.rejects(fs.access(path.join(directory, `${id}.recovery`)), /ENOENT/) + } finally { + await manager.shutdown() + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workspaces/__tests__/git-worktrees.test.ts b/packages/server/src/workspaces/__tests__/git-worktrees.test.ts index bc6382a1c..b363ca9f6 100644 --- a/packages/server/src/workspaces/__tests__/git-worktrees.test.ts +++ b/packages/server/src/workspaces/__tests__/git-worktrees.test.ts @@ -1,9 +1,42 @@ import assert from "node:assert/strict" -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import path from "node:path" import { describe, it } from "node:test" -import { listWorktrees } from "../git-worktrees" +import { createManagedWorktree, isValidWorktreeSlug, listWorktrees, resolveRepoRoot } from "../git-worktrees" + +async function waitForFile(file: string): Promise { + const deadline = Date.now() + 2_000 + while (!existsSync(file) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.equal(existsSync(file), true, `Timed out waiting for ${file}`) +} + +function installHangingGit(temp: string): { binDir: string; started: string; survived: string } { + const binDir = path.join(temp, "bin") + const script = path.join(temp, "hanging-git.cjs") + const started = path.join(temp, "started") + const survived = path.join(temp, "descendant-survived") + mkdirSync(binDir, { recursive: true }) + writeFileSync(script, [ + 'const { spawn } = require("node:child_process")', + 'const { writeFileSync } = require("node:fs")', + `writeFileSync(${JSON.stringify(started)}, "started")`, + "process.on(\"SIGTERM\", () => {})", + `spawn(process.execPath, ["-e", ${JSON.stringify(`process.on("SIGTERM", () => {}); setTimeout(() => require("node:fs").writeFileSync(${JSON.stringify(survived)}, "survived"), 700); setInterval(() => {}, 10_000)`) }], { stdio: "ignore", detached: process.platform === "darwin" })`, + "setInterval(() => {}, 10_000)", + ].join("\n")) + + const gitPath = path.join(binDir, process.platform === "win32" ? "git.cmd" : "git") + if (process.platform === "win32") { + writeFileSync(gitPath, `@echo off\r\n"${process.execPath}" "${script}" %*\r\n`) + } else { + const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'` + writeFileSync(gitPath, `#!/bin/sh\nexec ${quote(process.execPath)} ${quote(script)} "$@"\n`, { mode: 0o755 }) + } + return { binDir, started, survived } +} describe("listWorktrees", () => { it("uses the selected workspace folder for the root worktree directory", async () => { @@ -26,7 +59,7 @@ describe("listWorktrees", () => { ].join("\n") if (process.platform === "win32") { - writeFileSync(gitPath, `@echo off\r\nif "%1"=="worktree" if "%2"=="list" if "%3"=="--porcelain" (\r\necho ${porcelain.replace(/\n/g, "\r\necho ")}\r\nexit /b 0\r\n)\r\nexit /b 1\r\n`) + writeFileSync(gitPath, `@echo off\r\nif "%~1"=="worktree" if "%~2"=="list" if "%~3"=="--porcelain" (\r\necho ${porcelain.replace(/\n/g, "\r\necho ")}\r\nexit /b 0\r\n)\r\nexit /b 1\r\n`) } else { writeFileSync(gitPath, `#!/bin/sh\nif [ "$1" = "worktree" ] && [ "$2" = "list" ] && [ "$3" = "--porcelain" ]; then\nprintf '%s\n' '${porcelain.replace(/'/g, "'\\''")}'\nexit 0\nfi\nexit 1\n`, { mode: 0o755 }) } @@ -46,3 +79,72 @@ describe("listWorktrees", () => { } }) }) + +describe("resolveRepoRoot cancellation", () => { + for (const boundary of ["abort", "timeout"] as const) { + it(`terminates the complete git process tree on ${boundary} and waits for close`, async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-git-cancel-")) + const originalPath = process.env.PATH + const { binDir, started, survived } = installHangingGit(temp) + const controller = new AbortController() + + try { + process.env.PATH = `${binDir}${path.delimiter}${originalPath ?? ""}` + const operation = resolveRepoRoot(temp, undefined, { + signal: controller.signal, + timeoutMs: boundary === "timeout" ? 40 : 5_000, + }) + await waitForFile(started) + if (boundary === "abort") controller.abort(new Error("abort requested")) + + await assert.rejects(operation, boundary === "abort" ? /abort requested/ : /timed out after 40ms/) + await new Promise((resolve) => setTimeout(resolve, 850)) + assert.equal(existsSync(survived), false, "git helper descendant continued after cancellation completed") + } finally { + process.env.PATH = originalPath + rmSync(temp, { recursive: true, force: true }) + } + }) + } +}) + +describe("createManagedWorktree", () => { + it("rejects slugs that can alter a Windows shell command", () => { + assert.equal(isValidWorktreeSlug("feature/review-1.2"), true) + for (const slug of [ + "-config", + "review branch", + "review&calc", + "review|calc", + "review>file", + "review { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-worktree-")) + const repoRoot = path.join(temp, "repo") + const outside = path.join(temp, "outside") + + try { + mkdirSync(path.join(repoRoot, ".codenomad"), { recursive: true }) + mkdirSync(outside, { recursive: true }) + symlinkSync(outside, path.join(repoRoot, ".codenomad", "worktrees"), process.platform === "win32" ? "junction" : "dir") + + await assert.rejects( + createManagedWorktree({ repoRoot, workspaceFolder: repoRoot, slug: "escaped" }), + /escapes repository/, + ) + assert.equal(existsSync(path.join(outside, "escaped")), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/server/src/workspaces/__tests__/spawn.test.ts b/packages/server/src/workspaces/__tests__/spawn.test.ts index d11d8a66d..e4fec0ebb 100644 --- a/packages/server/src/workspaces/__tests__/spawn.test.ts +++ b/packages/server/src/workspaces/__tests__/spawn.test.ts @@ -1,7 +1,8 @@ import assert from "node:assert/strict" -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { tmpdir } from "node:os" import path from "node:path" +import { spawnSync } from "node:child_process" import { describe, it } from "node:test" import { buildWindowsSpawnSpec, parseWslUncPath, resolveWslWorkingDirectory } from "../spawn" @@ -78,7 +79,7 @@ describe("buildWindowsSpawnSpec", () => { assert.equal(spec.command, "test-cmd.exe") assert.equal(spec.processKind, "windows-wrapper") assert.equal(spec.options.windowsVerbatimArguments, true) - assert.match(spec.args[3] ?? "", new RegExp(escapeRegex(path.win32.resolve(shim)), "i")) + assert.match((spec.args[4] ?? "").replace(/\^/g, ""), new RegExp(escapeRegex(path.win32.resolve(shim)), "i")) } finally { rmSync(root, { recursive: true, force: true }) } @@ -114,6 +115,37 @@ describe("buildWindowsSpawnSpec", () => { assert.equal(spec.options.windowsVerbatimArguments, undefined) }) + it("passes spaces and shell metacharacters to cmd wrappers as literal arguments", { skip: process.platform !== "win32" }, () => { + const root = mkdtempSync(path.join(tmpdir(), "codenomad spawn ")) + const capture = path.join(root, "capture.cjs") + const injected = path.join(root, "injected") + const unsafeArg = `review&echo injected>${injected}` + mkdirSync(path.join(root, "bin with spaces"), { recursive: true }) + writeFileSync(capture, 'require("node:fs").writeFileSync(process.env.CAPTURE_OUTPUT, JSON.stringify(process.argv.slice(2)))\n') + + try { + for (const extension of ["cmd", "bat"]) { + const wrapper = path.join(root, "bin with spaces", `git.${extension}`) + const output = path.join(root, `args-${extension}.json`) + writeFileSync(wrapper, `@echo off\r\n"${process.execPath}" "${capture}" %*\r\n`) + const spec = buildWindowsSpawnSpec(wrapper, ["worktree", "path with spaces", unsafeArg], { + env: { ...process.env, CAPTURE_OUTPUT: output }, + }) + const result = spawnSync(spec.command, spec.args, { + env: spec.env, + encoding: "utf8", + windowsVerbatimArguments: spec.options.windowsVerbatimArguments, + }) + + assert.equal(result.status, 0, result.stderr) + assert.equal(existsSync(injected), false) + assert.deepEqual(JSON.parse(readFileSync(output, "utf8")), ["worktree", "path with spaces", unsafeArg]) + } + } finally { + rmSync(root, { recursive: true, force: true }) + } + }) + it("wraps WSL binaries with wsl.exe and propagates required env vars", () => { const spec = buildWindowsSpawnSpec( String.raw`\\wsl.localhost\Ubuntu\home\dev\.opencode\bin\opencode`, diff --git a/packages/server/src/workspaces/__tests__/workspace-identity.test.ts b/packages/server/src/workspaces/__tests__/workspace-identity.test.ts index ccf8e6ca4..59e569b2f 100644 --- a/packages/server/src/workspaces/__tests__/workspace-identity.test.ts +++ b/packages/server/src/workspaces/__tests__/workspace-identity.test.ts @@ -183,4 +183,17 @@ describe("workspace identity", () => { assert.equal(reused.created, false) assert.equal(reused.workspace.id, normal.workspace.id) }) + + it("uses an explicit lineage before reusing a workspace by path", async () => { + const { root, target, link } = await createLinkedWorkspace() + const manager = createManager(root) + const normal = await manager.create(target) + const restored = await manager.create(link, undefined, { lineageId: "restored-lineage" }) + const repeated = await manager.create(target, undefined, { lineageId: "restored-lineage" }) + + assert.notEqual(restored.workspace.id, normal.workspace.id) + assert.equal(restored.workspace.lineageId, "restored-lineage") + assert.equal(repeated.workspace.id, restored.workspace.id) + assert.equal(repeated.created, false) + }) }) diff --git a/packages/server/src/workspaces/git-worktrees.ts b/packages/server/src/workspaces/git-worktrees.ts index 087009015..81b152e8b 100644 --- a/packages/server/src/workspaces/git-worktrees.ts +++ b/packages/server/src/workspaces/git-worktrees.ts @@ -1,7 +1,22 @@ import path from "path" -import { spawn } from "child_process" +import { spawn, spawnSync, type ChildProcess } from "child_process" +import { randomBytes } from "node:crypto" import type { WorktreeDescriptor } from "../api-types" import { promises as fsp } from "fs" +import { buildSpawnSpec } from "./spawn" +import { + LAUNCH_CLEANUP_TOKEN_ENV, + probeLaunchCleanupToken, + probePosixProcesses, + signalLaunchCleanupToken, + signalOwnedPosixProcessGroup, + signalPosixProcesses, +} from "./process-identity" + +const DEFAULT_GIT_TIMEOUT_MS = 30_000 +const GIT_GRACEFUL_STOP_MS = 250 +const GIT_CLEANUP_TIMEOUT_MS = 2_000 +const GIT_CLEANUP_COMMAND_TIMEOUT_MS = 500 export interface LogLike { debug?: (obj: any, msg?: string) => void @@ -9,16 +24,158 @@ export interface LogLike { } type GitResult = { ok: true; stdout: string } | { ok: false; error: Error; stdout?: string; stderr?: string } +type GitRunOptions = { signal?: AbortSignal; timeoutMs?: number } function isGitUnavailableResult(result: GitResult): boolean { return !result.ok && (result.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT" } -function runGit(args: string[], cwd: string): Promise { - return new Promise((resolve) => { - const child = spawn("git", args, { cwd, stdio: ["ignore", "pipe", "pipe"] }) +async function terminateGitProcessTree(child: ChildProcess, cleanupToken: string): Promise { + const pid = child.pid + if (!pid) throw new Error("spawned git process did not expose a PID") + + const deadline = Date.now() + GIT_CLEANUP_TIMEOUT_MS + const failures: string[] = [] + let closed = child.exitCode !== null || child.signalCode !== null + const onClose = () => { closed = true } + child.once("close", onClose) + + const remainingCommandTime = () => Math.max(1, Math.min(GIT_CLEANUP_COMMAND_TIMEOUT_MS, deadline - Date.now())) + const waitUntil = async (condition: () => boolean, until: number): Promise => { + while (Date.now() < until) { + if (condition()) return true + await new Promise((resolve) => setTimeout(resolve, 20)) + } + return condition() + } + + try { + if (process.platform === "win32") { + let treeCleanupConfirmed = false + const stopTree = (force: boolean) => { + if (Date.now() >= deadline) return + try { + const result = spawnSync( + "taskkill.exe", + ["/PID", String(pid), "/T", ...(force ? ["/F"] : [])], + { encoding: "utf8", timeout: remainingCommandTime() }, + ) + if (result.status === 0) treeCleanupConfirmed = true + else failures.push(String(result.stderr || result.stdout || result.error?.message || `exit code ${result.status}`).trim()) + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)) + } + } + + // Windows wrappers can exit before their descendants after a non-forced taskkill. + stopTree(true) + if (!await waitUntil(() => closed && treeCleanupConfirmed, deadline)) { + throw new Error(`git Windows process-tree cleanup was not confirmed before the cleanup deadline${failures.length ? `: ${failures.join("; ")}` : ""}`) + } + return + } + + const signalTree = (signal: NodeJS.Signals) => { + if (Date.now() >= deadline) return + const snapshot = probePosixProcesses(spawnSync, remainingCommandTime(), process.platform, { groupId: pid }) + const members = snapshot.ok + ? [...snapshot.processes.values()].filter((identity) => identity.groupId === pid) + : [] + const leader = members.find((identity) => identity.pid === pid) ?? members[0] + const result = leader + ? signalPosixProcesses(spawnSync, { + leader, + groupId: pid, + members, + signal, + allowLeaderlessGroup: true, + cleanupToken, + }, remainingCommandTime(), process.platform) + : signalOwnedPosixProcessGroup(spawnSync, pid, signal, remainingCommandTime()) + if (!result.ok) failures.push(`POSIX ${signal}: ${result.error}`) + if (process.platform === "linux" || process.platform === "darwin") { + const tokenResult = signalLaunchCleanupToken(spawnSync, cleanupToken, signal, remainingCommandTime(), undefined, process.platform) + if (!tokenResult.ok) failures.push(`launch-token ${signal}: ${tokenResult.error}`) + } + } + const treeGone = () => { + const snapshot = probePosixProcesses(spawnSync, remainingCommandTime(), process.platform, { groupId: pid }) + if (!snapshot.ok || [...snapshot.processes.values()].some((identity) => identity.groupId === pid)) return false + if (process.platform !== "linux" && process.platform !== "darwin") return true + const tokenSnapshot = probeLaunchCleanupToken(spawnSync, cleanupToken, remainingCommandTime(), undefined, process.platform) + return tokenSnapshot.ok && tokenSnapshot.processes.size === 0 + } + + signalTree("SIGTERM") + if (!await waitUntil(() => closed && treeGone(), Math.min(deadline, Date.now() + GIT_GRACEFUL_STOP_MS))) { + signalTree("SIGKILL") + } + if (!await waitUntil(() => closed && treeGone(), deadline)) { + throw new Error(`git POSIX process-tree cleanup was not confirmed before the cleanup deadline${failures.length ? `: ${failures.join("; ")}` : ""}`) + } + } finally { + child.removeListener("close", onClose) + } +} + +function runGit(args: string[], cwd: string, options: GitRunOptions = {}): Promise { + return new Promise((resolve, reject) => { + options.signal?.throwIfAborted() + const cleanupToken = randomBytes(32).toString("hex") + const spec = buildSpawnSpec("git", args, { + cwd, + env: { ...process.env, [LAUNCH_CLEANUP_TOKEN_ENV]: cleanupToken }, + }) + const child = spawn(spec.command, spec.args, { + cwd: spec.cwd, + env: spec.env, + stdio: ["ignore", "pipe", "pipe"], + detached: process.platform !== "win32", + windowsVerbatimArguments: Boolean(spec.options.windowsVerbatimArguments), + }) let stdout = "" let stderr = "" + let settled = false + let terminating = false + let timeout: ReturnType | undefined + const requestedTimeout = options.timeoutMs ?? DEFAULT_GIT_TIMEOUT_MS + const timeoutMs = Number.isFinite(requestedTimeout) ? Math.max(1, requestedTimeout) : DEFAULT_GIT_TIMEOUT_MS + + const cleanup = () => { + if (timeout) clearTimeout(timeout) + options.signal?.removeEventListener("abort", abort) + } + const finish = (result: GitResult) => { + if (settled) return + settled = true + cleanup() + resolve(result) + } + const terminate = (error: unknown) => { + if (settled || terminating) return + terminating = true + cleanup() + void terminateGitProcessTree(child, cleanupToken).then( + () => { + settled = true + reject(error) + }, + (cleanupError) => { + settled = true + const reason = error instanceof Error ? error : new Error(String(error)) + const combined = new Error(`${reason.message}; ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`) + ;(combined as Error & { cause?: unknown }).cause = new AggregateError([reason, cleanupError]) + reject(combined) + }, + ) + } + const abort = () => terminate(options.signal?.reason ?? new Error("Git operation aborted")) + timeout = setTimeout( + () => terminate(new Error(`git ${args.join(" ")} timed out after ${timeoutMs}ms`)), + timeoutMs, + ) + options.signal?.addEventListener("abort", abort, { once: true }) + if (options.signal?.aborted) abort() child.stdout?.on("data", (chunk) => { stdout += chunk.toString() @@ -27,21 +184,27 @@ function runGit(args: string[], cwd: string): Promise { stderr += chunk.toString() }) child.once("error", (error) => { - resolve({ ok: false, error, stdout, stderr }) + if (terminating) return + finish({ ok: false, error, stdout, stderr }) }) child.once("close", (code) => { + if (terminating) return if (code === 0) { - resolve({ ok: true, stdout }) + finish({ ok: true, stdout }) } else { const error = new Error(stderr.trim() || `git ${args.join(" ")} failed with code ${code}`) - resolve({ ok: false, error, stdout, stderr }) + finish({ ok: false, error, stdout, stderr }) } }) }) } -export async function resolveRepoRoot(folder: string, logger?: LogLike): Promise<{ repoRoot: string; isGitRepo: boolean }> { - const result = await runGit(["rev-parse", "--show-toplevel"], folder) +export async function resolveRepoRoot( + folder: string, + logger?: LogLike, + options: GitRunOptions = {}, +): Promise<{ repoRoot: string; isGitRepo: boolean }> { + const result = await runGit(["rev-parse", "--show-toplevel"], folder, options) if (isGitUnavailableResult(result)) { throw new Error("Git is not installed or not available in PATH") } @@ -100,10 +263,12 @@ export async function listWorktrees(params: { repoRoot: string workspaceFolder: string logger?: LogLike + signal?: AbortSignal + timeoutMs?: number }): Promise { const { repoRoot, workspaceFolder, logger } = params - const result = await runGit(["worktree", "list", "--porcelain"], workspaceFolder) + const result = await runGit(["worktree", "list", "--porcelain"], workspaceFolder, params) if (!result.ok) { const rootDescriptor: WorktreeDescriptor = { slug: "root", directory: workspaceFolder, kind: "root" } logger?.debug?.({ repoRoot, err: result.error }, "Failed to list git worktrees; returning root only") @@ -161,13 +326,41 @@ export async function listWorktrees(params: { } export function isValidWorktreeSlug(slug: string): boolean { - if (!slug) return false - const trimmed = slug.trim() - if (!trimmed) return false - if (trimmed.length > 200) return false - // Disallow control characters; allow branch-like slugs including '/'. - if (/[\x00-\x1F\x7F]/.test(trimmed)) return false - return true + if (!slug || slug.length > 200) return false + return /^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/.test(slug) +} + +export function getManagedWorktreePath(repoRoot: string, slug: string): string { + const directoryName = slug + .trim() + .replace(/[\\/]+/g, "-") + .replace(/\s+/g, "-") + .replace(/[^a-zA-Z0-9_.-]+/g, "-") + .replace(/-{2,}/g, "-") + .replace(/^-+|-+$/g, "") || "worktree" + return path.join(repoRoot, ".codenomad", "worktrees", directoryName) +} + +export async function isManagedWorktree(params: { + repoRoot: string + worktree: WorktreeDescriptor +}): Promise { + if (params.worktree.kind !== "worktree") return false + try { + const repoRoot = await fsp.realpath(params.repoRoot) + const managedRoot = await fsp.realpath(path.join(params.repoRoot, ".codenomad", "worktrees")) + const directory = await fsp.realpath(params.worktree.directory) + const normalize = (value: string) => process.platform === "win32" ? value.toLowerCase() : value + if (normalize(managedRoot) !== normalize(path.join(repoRoot, ".codenomad", "worktrees"))) return false + if (normalize(path.dirname(directory)) !== normalize(managedRoot)) return false + + const metadata = await fsp.readFile(path.join(directory, ".git"), "utf8") + const gitDir = metadata.match(/^gitdir:\s*(.+)$/m)?.[1]?.trim() + if (!gitDir) return false + return (await fsp.stat(path.resolve(directory, gitDir))).isDirectory() + } catch { + return false + } } export async function createManagedWorktree(params: { @@ -175,6 +368,8 @@ export async function createManagedWorktree(params: { workspaceFolder: string slug: string logger?: LogLike + signal?: AbortSignal + timeoutMs?: number }): Promise<{ slug: string; directory: string; branch?: string }> { const { repoRoot, workspaceFolder, logger } = params const branch = params.slug.trim() @@ -183,26 +378,32 @@ export async function createManagedWorktree(params: { throw new Error("Invalid worktree slug") } - const sanitizeDirName = (input: string): string => { - const normalized = input - .trim() - .replace(/[\\/]+/g, "-") - .replace(/\s+/g, "-") - .replace(/[^a-zA-Z0-9_.-]+/g, "-") - .replace(/-{2,}/g, "-") - .replace(/^-+|-+$/g, "") - return normalized || "worktree" + const canonicalRepoRoot = await fsp.realpath(repoRoot) + const codenomadDir = path.join(repoRoot, ".codenomad") + const worktreesDir = path.join(codenomadDir, "worktrees") + const pathsEqual = (left: string, right: string) => process.platform === "win32" + ? left.toLowerCase() === right.toLowerCase() + : left === right + + await fsp.mkdir(codenomadDir, { recursive: true }) + const canonicalCodenomadDir = await fsp.realpath(codenomadDir) + if (!pathsEqual(canonicalCodenomadDir, path.join(canonicalRepoRoot, ".codenomad"))) { + throw new Error("Managed worktree directory escapes repository") } - - const worktreesDir = path.join(repoRoot, ".codenomad", "worktrees") - const targetDir = path.join(worktreesDir, sanitizeDirName(branch)) await fsp.mkdir(worktreesDir, { recursive: true }) + const canonicalWorktreesDir = await fsp.realpath(worktreesDir) + if (!pathsEqual(canonicalWorktreesDir, path.join(canonicalCodenomadDir, "worktrees"))) { + throw new Error("Managed worktree directory escapes repository") + } + const targetDir = getManagedWorktreePath(canonicalRepoRoot, branch) + const relativeTarget = path.relative(canonicalWorktreesDir, targetDir) + if (!relativeTarget || relativeTarget.startsWith(`..${path.sep}`) || path.isAbsolute(relativeTarget)) { + throw new Error("Managed worktree target escapes managed directory") + } try { - const stat = await fsp.stat(targetDir) - if (stat.isDirectory()) { - throw new Error("Worktree directory already exists") - } + await fsp.lstat(targetDir) + throw new Error("Worktree directory already exists") } catch (error) { const code = (error as NodeJS.ErrnoException).code if (code !== "ENOENT") { @@ -213,7 +414,7 @@ export async function createManagedWorktree(params: { logger?.debug?.({ slug: branch, branch, targetDir }, "Creating managed git worktree") // Prefer creating a new branch from HEAD. - const first = await runGit(["worktree", "add", "-b", branch, targetDir, "HEAD"], workspaceFolder) + const first = await runGit(["worktree", "add", "-b", branch, targetDir, "HEAD"], workspaceFolder, params) if (first.ok) { return { slug: branch, directory: targetDir, branch } } @@ -221,7 +422,7 @@ export async function createManagedWorktree(params: { const message = first.stderr?.toLowerCase() ?? first.error.message.toLowerCase() if (message.includes("already exists")) { // If the branch already exists, add worktree for that branch. - const second = await runGit(["worktree", "add", targetDir, branch], workspaceFolder) + const second = await runGit(["worktree", "add", targetDir, branch], workspaceFolder, params) if (second.ok) { return { slug: branch, directory: targetDir, branch } } diff --git a/packages/server/src/workspaces/instance-client.test.ts b/packages/server/src/workspaces/instance-client.test.ts index 636bc9f1a..20da6729c 100644 --- a/packages/server/src/workspaces/instance-client.test.ts +++ b/packages/server/src/workspaces/instance-client.test.ts @@ -148,8 +148,9 @@ describe("createInstanceClient", () => { } }) - it("applies the loopback timeout and aborts a stuck instance", async () => { + it("applies the loopback timeout and aborts a stuck instance", { timeout: 1_000 }, async () => { const original = globalThis.fetch + const keepEventLoopAlive = setTimeout(() => {}, 2_000) // Never resolves on its own; only settles when the passed signal aborts, // mirroring how a real fetch honours an AbortSignal. Without the factory // timeout this call would hang forever and time the test out. @@ -169,7 +170,71 @@ describe("createInstanceClient", () => { const result = await client!.global.health() assert.ok(result.error, "expected the stuck-instance call to surface an error") } finally { + clearTimeout(keepEventLoopAlive) globalThis.fetch = original } }) + + it("preserves the SDK Request signal while retaining the fallback timeout", async () => { + const originalFetch = globalThis.fetch + let effectiveSignal: AbortSignal | null | undefined + globalThis.fetch = ((_input, init) => { + effectiveSignal = init?.signal + return new Promise((_resolve, reject) => effectiveSignal?.addEventListener("abort", () => reject(effectiveSignal?.reason), { once: true })) + }) as typeof fetch + try { + const manager = makeManager({ getInstancePort: () => 4321, get: () => ({ path: "/repo" }) }) + const client = createInstanceClient(manager as unknown as WorkspaceManager, "ws-1", { timeoutMs: 1_000 })! + const controller = new AbortController() + const request = client.tool.ids(undefined, { signal: controller.signal }) + while (!effectiveSignal) await new Promise((resolve) => setImmediate(resolve)) + const reason = new Error("caller cancelled") + controller.abort(reason) + assert.equal(effectiveSignal.aborted, true) + assert.strictEqual(effectiveSignal.reason, reason) + await request.catch(() => undefined) + } finally { + globalThis.fetch = originalFetch + } + }) + + it("preserves a caller signal that is aborted before dispatch", { timeout: 1_000 }, async () => { + const originalFetch = globalThis.fetch + let effectiveSignal: AbortSignal | null | undefined + globalThis.fetch = ((_input, init) => { + effectiveSignal = init?.signal + if (effectiveSignal?.aborted) return Promise.reject(effectiveSignal.reason) + return new Promise((_resolve, reject) => effectiveSignal?.addEventListener("abort", () => reject(effectiveSignal?.reason), { once: true })) + }) as typeof fetch + try { + const manager = makeManager({ getInstancePort: () => 4321, get: () => ({ path: "/repo" }) }) + const client = createInstanceClient(manager as unknown as WorkspaceManager, "ws-1", { timeoutMs: 1_000 })! + const controller = new AbortController() + const reason = new Error("cancelled before dispatch") + controller.abort(reason) + await client.tool.ids(undefined, { signal: controller.signal }).catch(() => undefined) + assert.equal(effectiveSignal?.aborted, true) + assert.strictEqual(effectiveSignal?.reason, reason) + } finally { + globalThis.fetch = originalFetch + } + }) + + it("times out requests even when the Request carries its default signal", { timeout: 1_000 }, async () => { + const originalFetch = globalThis.fetch + let effectiveSignal: AbortSignal | null | undefined + globalThis.fetch = ((_input, init) => { + effectiveSignal = init?.signal + return new Promise((_resolve, reject) => effectiveSignal?.addEventListener("abort", () => reject(effectiveSignal?.reason), { once: true })) + }) as typeof fetch + try { + const manager = makeManager({ getInstancePort: () => 4321, get: () => ({ path: "/repo" }) }) + const request = createInstanceClient(manager as unknown as WorkspaceManager, "ws-1", { timeoutMs: 5 })!.tool.ids() + while (!effectiveSignal?.aborted) await new Promise((resolve) => setTimeout(resolve, 1)) + assert.equal(effectiveSignal.reason?.name, "TimeoutError") + await request.catch(() => undefined) + } finally { + globalThis.fetch = originalFetch + } + }) }) diff --git a/packages/server/src/workspaces/instance-client.ts b/packages/server/src/workspaces/instance-client.ts index 5ee323f04..40bed72c3 100644 --- a/packages/server/src/workspaces/instance-client.ts +++ b/packages/server/src/workspaces/instance-client.ts @@ -53,11 +53,14 @@ export function createInstanceClient( return createOpencodeClient({ baseUrl: `http://${LOOPBACK_HOST}:${port}/`, headers, - fetch: (url, init) => - fetch(url, { - ...(init as RequestInit), - signal: (init as RequestInit)?.signal ?? AbortSignal.timeout(timeoutMs), - }), + fetch: (url, init) => fetch(url, { + ...(init as RequestInit), + signal: AbortSignal.any([ + ...(url instanceof Request ? [url.signal] : []), + ...((init as RequestInit | undefined)?.signal ? [(init as RequestInit).signal!] : []), + AbortSignal.timeout(timeoutMs), + ]), + }), ...(directory ? { directory } : {}), }) } diff --git a/packages/server/src/workspaces/launch-cleanup.test.ts b/packages/server/src/workspaces/launch-cleanup.test.ts index 2a2f34f8c..9d3a21dd2 100644 --- a/packages/server/src/workspaces/launch-cleanup.test.ts +++ b/packages/server/src/workspaces/launch-cleanup.test.ts @@ -5,6 +5,7 @@ import { LAUNCH_CLEANUP_TOKEN_ENV, probeLaunchCleanupToken, signalLaunchCleanupT type Spawn = typeof import("node:child_process").spawnSync const result = (stdout: string): SpawnSyncReturns => ({ pid: 1, output: [null, stdout, ""], stdout, stderr: "", status: 0, signal: null }) +const b64 = (value: string) => Buffer.from(value).toString("base64") describe("launch cleanup token adapter", () => { it("passes the exact token to the bounded Linux environ probe", () => { @@ -31,4 +32,40 @@ describe("launch cleanup token adapter", () => { const malformed = ((() => result("5000|1|4242|150|boot-a|150|truncated\n")) as unknown) as Spawn assert.equal(probeLaunchCleanupToken(malformed, "c".repeat(64), 25).ok, false) }) + + it("probes and signals escaped cleanup-token processes with portable POSIX ps", () => { + const token = "d".repeat(64) + const processRow = `CODENOMAD_PROCESS_B64|5000|1|9000|${b64("Tue Aug 4 12:00:00 2026")}|${b64("git helper")}\n` + const probeCalls: any[] = [] + const probeRun = ((command: string, args: string[], options: object) => { + probeCalls.push(command, args, options) + return result(processRow) + }) as unknown as Spawn + const probe = probeLaunchCleanupToken(probeRun, token, 25, undefined, "darwin") + assert.equal(probe.ok && probe.processes.get(5000)?.groupId, 9000) + assert.equal(probeCalls[0], "sh") + assert.match(probeCalls[1][1], /ps -axo pid=/) + assert.match(probeCalls[1][1], /ps eww -p/) + assert.equal(probeCalls[1].includes(token), true) + + const targetRow = processRow.replace("CODENOMAD_PROCESS_B64", "CODENOMAD_TARGET_B64") + const cleanup = signalLaunchCleanupToken( + ((() => result(`${targetRow}CODENOMAD_RESULT|1\n`)) as unknown) as Spawn, + token, "SIGKILL", 25, undefined, "darwin", + ) + assert.deepEqual([cleanup.ok, cleanup.signalSent, cleanup.targets[0]?.pid], [true, true, 5000]) + }) + + it("never searches Windows command lines for the environment-only cleanup token", () => { + const calls: any[] = [] + const run = ((command: string, args: string[], options: object) => { + calls.push(command, args, options) + return result("") + }) as unknown as Spawn + const token = "e".repeat(64) + const probe = probeLaunchCleanupToken(run, token, 25, undefined, "win32") + assert.equal(probe.ok, false) + assert.match(!probe.ok ? probe.error : "", /persisted launch-tree identities/) + assert.deepEqual(calls, []) + }) }) diff --git a/packages/server/src/workspaces/managed-worktree.test.ts b/packages/server/src/workspaces/managed-worktree.test.ts new file mode 100644 index 000000000..dae74bb21 --- /dev/null +++ b/packages/server/src/workspaces/managed-worktree.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict" +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import { test } from "node:test" + +import { isManagedWorktree } from "./git-worktrees" + +test("managed worktrees require a canonical direct child with Git metadata", async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-worktree-")) + const repoRoot = path.join(temp, "repo") + const managed = path.join(repoRoot, ".codenomad", "worktrees", "review") + const metadata = path.join(repoRoot, ".git", "worktrees", "review") + const external = path.join(repoRoot, "..", "external") + try { + mkdirSync(managed, { recursive: true }) + mkdirSync(metadata, { recursive: true }) + mkdirSync(external, { recursive: true }) + writeFileSync(path.join(managed, ".git"), `gitdir: ${metadata}\n`) + + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: managed, kind: "worktree" }, + }), true) + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: external, kind: "worktree" }, + }), false) + rmSync(path.join(managed, ".git")) + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory: managed, kind: "worktree" }, + }), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) + +test("managed worktree root cannot redirect outside the repository", async () => { + const temp = mkdtempSync(path.join(tmpdir(), "codenomad-managed-root-")) + const repoRoot = path.join(temp, "repo") + const externalRoot = path.join(temp, "external-worktrees") + const directory = path.join(externalRoot, "review") + const metadata = path.join(repoRoot, ".git", "worktrees", "review") + try { + mkdirSync(path.join(repoRoot, ".codenomad"), { recursive: true }) + mkdirSync(directory, { recursive: true }) + mkdirSync(metadata, { recursive: true }) + writeFileSync(path.join(directory, ".git"), `gitdir: ${metadata}\n`) + symlinkSync(externalRoot, path.join(repoRoot, ".codenomad", "worktrees"), "junction") + + assert.equal(await isManagedWorktree({ + repoRoot, + worktree: { slug: "review", directory, kind: "worktree" }, + }), false) + } finally { + rmSync(temp, { recursive: true, force: true }) + } +}) diff --git a/packages/server/src/workspaces/manager.test.ts b/packages/server/src/workspaces/manager.test.ts index e7f6061ac..f1b0beffd 100644 --- a/packages/server/src/workspaces/manager.test.ts +++ b/packages/server/src/workspaces/manager.test.ts @@ -1,4 +1,7 @@ import assert from "node:assert/strict" +import { mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" import { describe, it } from "node:test" import pino from "pino" @@ -10,9 +13,11 @@ import { } from "./runtime" import { WorkspaceCleanupTimeoutError, + WorkspaceDeletionBlockedError, WorkspaceLaunchCancelledError, WorkspaceLaunchTimeoutError, WorkspaceManager, + WorkspacePathOwnedError, WorkspaceShutdownError, } from "./manager" @@ -33,8 +38,10 @@ class ControlledRuntime { stopCalls = 0 failStops = 0 onExit?: (info: ProcessExitInfo) => void + launchEnvironment?: Record launch: WorkspaceRuntime["launch"] = (options) => { + this.launchEnvironment = options.environment this.active.add(options.workspaceId) this.onExit = options.onExit this.launchCalled.resolve(options.workspaceId) @@ -64,6 +71,7 @@ function createHarness(options: { launchTimeoutMs?: number setTimeout?: (callback: () => void, delayMs: number) => ReturnType clearTimeout?: (timer: ReturnType) => void + workspaceLeaseDir?: string } = {}) { const { stubReadiness = true, ...managerOptions } = options const eventBus = new EventBus() @@ -106,6 +114,174 @@ async function createReady(harness: ReturnType) { } describe("workspace manager lifecycle", () => { + it("blocks deletion after startup reserves an unpublished or retained error workspace", async () => { + const harness = createHarness() + const creation = harness.manager.create(process.cwd()) + const workspaceId = await harness.runtime.launchCalled.promise + + assert.equal(harness.manager.list().length, 0) + await harness.manager.withWorkspacePathLease(process.cwd(), async (active) => { + assert.equal(active, true) + }) + const record = (harness.manager as any).workspaces.get(workspaceId) + record.status = "error" + await harness.manager.withWorkspacePathLease(process.cwd(), async (active) => { + assert.equal(active, true) + }) + record.status = "starting" + await harness.manager.delete(workspaceId) + await assert.rejects(creation, WorkspaceLaunchCancelledError) + }) + + it("waits for a delete-first path lease before reserving startup", async () => { + const harness = createHarness() + const leaseEntered = deferred() + const releaseLease = deferred() + const deletion = harness.manager.withWorkspacePathLease(process.cwd(), async (active) => { + assert.equal(active, false) + leaseEntered.resolve() + await releaseLease.promise + }) + await leaseEntered.promise + + const creation = harness.manager.create(process.cwd()) + let launchStarted = false + void harness.runtime.launchCalled.promise.then(() => { launchStarted = true }) + await new Promise((resolve) => setImmediate(resolve)) + assert.equal(launchStarted, false) + + releaseLease.resolve() + await deletion + const workspaceId = await harness.runtime.launchCalled.promise + assert.equal(launchStarted, true) + await harness.manager.delete(workspaceId) + await assert.rejects(creation, WorkspaceLaunchCancelledError) + }) + + it("does not run startup after its queued path lease wait times out", async () => { + const harness = createHarness({ launchTimeoutMs: 25 }) + const leaseEntered = deferred() + const releaseLease = deferred() + const deletion = harness.manager.withWorkspacePathLease(process.cwd(), async () => { + leaseEntered.resolve() + await releaseLease.promise + }) + await leaseEntered.promise + + await assert.rejects(harness.manager.create(process.cwd()), WorkspaceLaunchTimeoutError) + releaseLease.resolve() + await deletion + await new Promise((resolve) => setImmediate(resolve)) + + assert.equal((harness.manager as any).workspaces.size, 0) + assert.equal(harness.runtime.active.size, 0) + }) + + it("guards restore cancellation before aborting or deleting its workspace", async () => { + const harness = createHarness() + const creation = harness.manager.create(process.cwd(), undefined, { requestId: "restore-request" }) + const workspaceId = await harness.runtime.launchCalled.promise + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + await creation + let guardedWorkspaceId: string | undefined + harness.manager.setDeletionGuard(async (workspace) => { + guardedWorkspaceId = workspace.id + throw new WorkspaceDeletionBlockedError(workspace.id) + }) + + await assert.rejects(harness.manager.cancelCreationRequest("restore-request"), WorkspaceDeletionBlockedError) + assert.equal(guardedWorkspaceId, workspaceId) + assert.equal(harness.manager.get(workspaceId)?.status, "ready") + assert.equal(harness.runtime.stopCalls, 0) + + harness.manager.setDeletionGuard(async (_workspace, operation) => operation()) + await harness.manager.cancelCreationRequest("restore-request") + assert.equal(harness.manager.get(workspaceId), undefined) + }) + + it("does not reuse a ready workspace while guarded deletion is pending", async () => { + const harness = createHarness() + const workspaceId = await createReady(harness) + const guardEntered = deferred() + const finishGuard = deferred() + harness.manager.setDeletionGuard(async (workspace) => { + guardEntered.resolve() + await finishGuard.promise + throw new WorkspaceDeletionBlockedError(workspace.id) + }) + + const deletion = harness.manager.delete(workspaceId) + await guardEntered.promise + await assert.rejects(harness.manager.create(process.cwd()), /deletion is pending/) + assert.equal((harness.manager as any).workspaces.size, 1) + assert.equal(harness.runtime.active.size, 1) + finishGuard.resolve() + await assert.rejects(deletion, WorkspaceDeletionBlockedError) + assert.equal(harness.manager.get(workspaceId)?.status, "ready") + + const reused = await harness.manager.create(process.cwd()) + assert.equal(reused.created, false) + assert.equal(reused.workspace.id, workspaceId) + }) + + it("does not reuse a pending workspace while its guarded deletion is pending", async () => { + const harness = createHarness() + const firstCreation = harness.manager.create(process.cwd()) + const workspaceId = await harness.runtime.launchCalled.promise + const guardEntered = deferred() + const finishGuard = deferred() + harness.manager.setDeletionGuard(async (_workspace, operation) => { + guardEntered.resolve() + await finishGuard.promise + return operation() + }) + + const deletion = harness.manager.delete(workspaceId) + await guardEntered.promise + await assert.rejects(harness.manager.create(process.cwd()), /deletion is pending/) + assert.equal((harness.manager as any).workspaces.size, 1) + assert.equal(harness.runtime.active.size, 1) + finishGuard.resolve() + await assert.rejects(firstCreation, WorkspaceLaunchCancelledError) + await deletion + }) + + for (const status of ["stopped", "error"] as const) { + it(`does not reuse a ${status} lineage record`, async () => { + const harness = createHarness() + const first = await (async () => { + const creation = harness.manager.create(process.cwd(), undefined, { lineageId: "terminal-lineage" }) + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + return creation + })() + ;(harness.manager.get(first.workspace.id) as { status: string }).status = status + + const second = await harness.manager.create(process.cwd(), undefined, { lineageId: "terminal-lineage" }) + + assert.equal(second.created, true) + assert.notEqual(second.workspace.id, first.workspace.id) + }) + } + + it("uses a distinct generated capability for plugin callbacks", async () => { + const harness = createHarness() + ;(harness.manager as any).options.settings = { + getOwner: () => ({ environmentVariables: { OPENCODE_SERVER_PASSWORD: "shared-opencode-secret" } }), + } + const workspaceId = await createReady(harness) + const callbackToken = harness.runtime.launchEnvironment?.CODENOMAD_CALLBACK_TOKEN + + assert.ok(callbackToken) + assert.notEqual(callbackToken, "shared-opencode-secret") + assert.equal(harness.manager.getPluginCallbackAuthorizationHeader(workspaceId), `Bearer ${callbackToken}`) + assert.equal( + harness.manager.getInstanceAuthorizationHeader(workspaceId), + `Basic ${Buffer.from("codenomad:shared-opencode-secret").toString("base64")}`, + ) + }) + it("rejects a healthy workspace whose OpenCode configuration is invalid", async () => { const originalFetch = globalThis.fetch const requests: string[] = [] @@ -182,12 +358,51 @@ describe("workspace manager lifecycle", () => { const failures = await Promise.allSettled([first, concurrent]) assert.deepEqual(failures.map((result) => result.status), ["rejected", "rejected"]) assert.equal(harness.runtime.active.has(workspaceId), true) + await assert.rejects(harness.manager.create(process.cwd()), /cleanup is incomplete/) await harness.manager.delete(workspaceId) assert.equal(harness.runtime.active.has(workspaceId), false) assert.equal(harness.manager.get(workspaceId), undefined) }) + it("blocks deletion and relaunch while stopped workspace cleanup is incomplete", async () => { + const harness = createHarness() + const workspaceId = await createReady(harness) + harness.runtime.active.delete(workspaceId) + harness.runtime.onExit?.({ workspaceId, code: 0, signal: null, requested: false }) + harness.runtime.failStops = 2 + + await assert.rejects(harness.manager.delete(workspaceId), /controlled stop failure/) + assert.equal(harness.manager.get(workspaceId)?.status, "stopped") + await harness.manager.withWorkspacePathLease(process.cwd(), async (active) => { + assert.equal(active, true) + }) + await assert.rejects(harness.manager.create(process.cwd()), /cleanup is incomplete/) + + await harness.manager.delete(workspaceId) + assert.equal(harness.manager.get(workspaceId), undefined) + }) + + it("exposes unpublished failed cleanup for safe deletion retry", async () => { + const harness = createHarness() + harness.runtime.failStops = 1 + const creation = harness.manager.create(process.cwd()) + const workspaceId = await harness.runtime.launchCalled.promise + harness.runtime.launchResult.reject(new Error("controlled launch failure")) + + await assert.rejects(creation, /controlled stop failure/) + assert.equal(harness.manager.list()[0]?.id, workspaceId) + assert.equal(harness.manager.get(workspaceId)?.status, "error") + await harness.manager.withWorkspacePathLease(process.cwd(), async (active) => { + assert.equal(active, true) + }) + await assert.rejects(harness.manager.create(process.cwd()), /cleanup is incomplete/) + + await harness.manager.delete(workspaceId) + assert.equal(harness.manager.get(workspaceId), undefined) + assert.equal(harness.runtime.active.has(workspaceId), false) + }) + it("retries cancellation deletion for an already-cancelled request", async () => { const harness = createHarness() const creation = harness.manager.create(process.cwd(), undefined, { requestId: "retry-cancel" }) @@ -290,6 +505,85 @@ describe("workspace manager lifecycle", () => { assert.equal(harness.manager.releaseCreationRequest(workspaceId, "restore-reused"), false) }) + it("retains a request-owned workspace when ordinary reuse adopts it", async () => { + const harness = createHarness() + const scoped = harness.manager.create(process.cwd(), undefined, { requestId: "old-restore" }) + const workspaceId = await harness.runtime.launchCalled.promise + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + await scoped + + const adopted = await harness.manager.create(process.cwd()) + assert.equal(adopted.workspace.id, workspaceId) + await harness.manager.cancelCreationRequest("old-restore") + assert.equal(harness.manager.get(workspaceId)?.id, workspaceId) + assert.equal(harness.runtime.active.has(workspaceId), true) + }) + + it("fences canonical paths across managers while preserving local forceNew", async () => { + const leaseDir = await mkdtemp(path.join(os.tmpdir(), "codenomad-workspace-leases-")) + try { + const first = createHarness({ workspaceLeaseDir: leaseDir }) + const second = createHarness({ workspaceLeaseDir: leaseDir }) + const creation = first.manager.create(process.cwd()) + const workspaceId = await first.runtime.launchCalled.promise + + await assert.rejects(second.manager.create(process.cwd()), WorkspacePathOwnedError) + assert.equal(second.runtime.active.size, 0) + await second.manager.withWorkspacePathLease(process.cwd(), async (active) => assert.equal(active, true)) + first.runtime.resolveLaunch() + first.readiness.resolve(undefined) + await creation + + const forced = await first.manager.create(process.cwd(), undefined, { forceNew: true }) + const forcedId = forced.workspace.id + assert.notEqual(forcedId, workspaceId) + + await first.manager.delete(forcedId) + first.runtime.failStops = 2 + await assert.rejects(first.manager.delete(workspaceId), /controlled stop failure/) + await assert.rejects(second.manager.create(process.cwd()), WorkspacePathOwnedError) + await first.manager.delete(workspaceId) + const replacement = second.manager.create(process.cwd()) + second.runtime.resolveLaunch() + second.readiness.resolve(undefined) + await replacement + await second.manager.shutdown() + } finally { + await rm(leaseDir, { recursive: true, force: true }) + } + }) + + it("fences and stops a workspace when its process lease is replaced", async () => { + const leaseDir = await mkdtemp(path.join(os.tmpdir(), "codenomad-workspace-lease-loss-")) + try { + const harness = createHarness({ workspaceLeaseDir: leaseDir }) + const creation = harness.manager.create(process.cwd()) + const workspaceId = await harness.runtime.launchCalled.promise + harness.runtime.resolveLaunch() + harness.readiness.resolve(undefined) + await creation + const registry = (harness.manager as any).processLeases + const [key, held] = [...registry.held.entries()][0] + await rename(path.join(held.directory, "owner"), path.join(held.directory, "retired.test-owner")) + await mkdir(path.join(held.directory, "owner")) + await writeFile(path.join(held.directory, "owner", "owner.json"), JSON.stringify({ + version: 1, managerToken: "successor", leaseToken: "successor-lease", pid: 999, + hostname: os.hostname(), workspacePath: process.cwd(), + }), "utf8") + + await registry.heartbeat(key, held.owner) + for (let attempt = 0; attempt < 50 && harness.manager.get(workspaceId); attempt += 1) { + await new Promise((resolve) => setImmediate(resolve)) + } + assert.equal(harness.manager.getInstancePort(workspaceId), undefined) + assert.equal(harness.runtime.active.has(workspaceId), false) + assert.equal(harness.manager.get(workspaceId), undefined) + } finally { + await rm(leaseDir, { recursive: true, force: true }) + } + }) + for (const boundary of ["runtime launch", "health readiness"] as const) { it(`applies one shared end-to-end deadline during ${boundary} and cleans up`, async () => { const deadlines: Array<() => void> = [] diff --git a/packages/server/src/workspaces/manager.ts b/packages/server/src/workspaces/manager.ts index 6d39c9192..a63e90d09 100644 --- a/packages/server/src/workspaces/manager.ts +++ b/packages/server/src/workspaces/manager.ts @@ -18,15 +18,22 @@ import { resolveExistingOpencodeConfigContent, } from "../opencode-plugin.js" import { + CODENOMAD_CALLBACK_TOKEN_ENV, OPENCODE_SERVER_BASE_URL_ENV, + buildCodeNomadCallbackAuthorizationHeader, buildOpencodeBasicAuthHeader, + generateCodeNomadCallbackToken, OPENCODE_SERVER_PASSWORD_ENV, OPENCODE_SERVER_USERNAME_ENV, resolveOpencodeServerAuth, } from "./opencode-auth" -import { resolveWorkspaceIdentity } from "./workspace-identity" +import { normalizeWorkspaceIdentityPath, resolveWorkspaceIdentity } from "./workspace-identity" import { parseWslUncPath } from "./spawn" import { LOOPBACK_HOST } from "./loopback" +import { + WorkspaceProcessLeaseRegistry, + type WorkspaceProcessLease, +} from "./process-lease" const STARTUP_STABILITY_DELAY_MS = 1500 const DEFAULT_LAUNCH_TIMEOUT_MS = 30_000 @@ -68,8 +75,14 @@ interface WorkspaceManagerOptions { launchTimeoutMs?: number setTimeout?: (callback: () => void, delayMs: number) => ManagerTimeout clearTimeout?: (timer: ManagerTimeout) => void + workspaceLeaseDir?: string } +export type WorkspaceDeletionGuard = ( + workspace: WorkspaceDescriptor, + operation: () => Promise, +) => Promise + interface WorkspaceRecord extends WorkspaceDescriptor { identityKey: string ownership: WorkspaceCreationOwnership @@ -83,6 +96,9 @@ interface WorkspaceState { deletePromise?: Promise published: boolean stoppedPublished: boolean + cleanupBlocked?: boolean + leaseLost?: boolean + processLease?: WorkspaceProcessLease } export class WorkspaceLaunchCancelledError extends Error { constructor(workspaceId: string) { @@ -106,6 +122,20 @@ export class WorkspaceCleanupTimeoutError extends Error { this.name = "WorkspaceCleanupTimeoutError" } } +export class WorkspaceDeletionBlockedError extends Error { + readonly code = "WORKSPACE_OWNED_BY_WORKFLOW" + constructor(workspaceId: string) { + super(`Workspace ${workspaceId} is owned by an active workflow`) + this.name = "WorkspaceDeletionBlockedError" + } +} +export class WorkspacePathOwnedError extends Error { + readonly code = "WORKSPACE_OWNED_BY_OTHER_SERVER" + constructor(workspacePath: string) { + super(`Workspace path is in use by another CodeNomad server: ${workspacePath}`) + this.name = "WorkspacePathOwnedError" + } +} export class WorkspaceShutdownError extends AggregateError { readonly code = "WORKSPACE_SHUTDOWN_FAILED" readonly retryable = true @@ -122,6 +152,7 @@ export interface WorkspaceCreateOptions { binaryPath?: string requestId?: string forceNew?: boolean + lineageId?: string } type CreationRequestState = "active" | "cancelled" | "released" type WorkspaceCreationOwnership = Map @@ -140,28 +171,93 @@ export class WorkspaceManager { private readonly runtime: Pick private readonly codeNomadPluginUrl: string private readonly opencodeAuth = new Map() + private readonly pluginCallbackAuth = new Map() + private readonly workspacePathLeases = new Map>() + private readonly processLeases?: WorkspaceProcessLeaseRegistry + private deletionGuard?: WorkspaceDeletionGuard constructor(private readonly options: WorkspaceManagerOptions) { this.runtime = options.runtime ?? new WorkspaceRuntime(this.options.eventBus, this.options.logger) this.codeNomadPluginUrl = getCodeNomadPluginUrl() + this.processLeases = options.workspaceLeaseDir + ? new WorkspaceProcessLeaseRegistry({ directory: options.workspaceLeaseDir }) + : undefined } list(): WorkspaceDescriptor[] { return Array.from(this.workspaces.values()) - .filter((record) => record[WORKSPACE_STATE].published) + .filter((record) => record[WORKSPACE_STATE].published || record[WORKSPACE_STATE].cleanupBlocked) } get(id: string): WorkspaceDescriptor | undefined { const record = this.workspaces.get(id) - return record?.[WORKSPACE_STATE].published ? record : undefined + return record && (record[WORKSPACE_STATE].published || record[WORKSPACE_STATE].cleanupBlocked) ? record : undefined } getInstancePort(id: string): number | undefined { const record = this.workspaces.get(id) - return record?.[WORKSPACE_STATE].published ? record.port : undefined + return record?.[WORKSPACE_STATE].published && !record[WORKSPACE_STATE].leaseLost ? record.port : undefined } getInstanceAuthorizationHeader(id: string): string | undefined { - return this.workspaces.get(id)?.[WORKSPACE_STATE].published ? this.opencodeAuth.get(id)?.authorization : undefined + const record = this.workspaces.get(id) + return record?.[WORKSPACE_STATE].published && !record[WORKSPACE_STATE].leaseLost ? this.opencodeAuth.get(id)?.authorization : undefined + } + + getPluginCallbackAuthorizationHeader(id: string): string | undefined { + const record = this.workspaces.get(id) + return record?.[WORKSPACE_STATE].published && !record[WORKSPACE_STATE].leaseLost ? this.pluginCallbackAuth.get(id) : undefined + } + + setDeletionGuard(guard: WorkspaceDeletionGuard): void { + this.deletionGuard = guard + } + + async withWorkspacePathLease( + folder: string, + operation: (active: boolean) => Promise, + ): Promise { + const { workspacePath } = await resolveWorkspaceIdentity(folder, this.options.rootDir) + const pathKey = normalizeWorkspaceIdentityPath(workspacePath) + return this.withWorkspacePathKeyLease(pathKey, async () => { + const active = Array.from(this.workspaces.values()).some((record) => + normalizeWorkspaceIdentityPath(record.path) === pathKey + && (record[WORKSPACE_STATE].cleanupBlocked + || record.status === "starting" || record.status === "ready" || record.status === "error")) + if (active || !this.processLeases) return operation(active) + const processLease = await this.processLeases.acquire(workspacePath) + if (!processLease) return operation(true) + try { + return await operation(false) + } finally { + await processLease.release() + } + }) + } + + private async withWorkspacePathKeyLease( + pathKey: string, + operation: () => T | Promise, + deadline?: { at: number; timeoutMs: number }, + ): Promise { + const previous = this.workspacePathLeases.get(pathKey) ?? Promise.resolve() + let release!: () => void + const hold = new Promise((resolve) => { release = resolve }) + const lease = previous.then(() => hold) + this.workspacePathLeases.set(pathKey, lease) + void lease.then(() => { + if (this.workspacePathLeases.get(pathKey) === lease) this.workspacePathLeases.delete(pathKey) + }) + try { + if (deadline) { + await this.withLaunchDeadline(previous, undefined, deadline.at, deadline.timeoutMs) + if (Date.now() >= deadline.at) throw new WorkspaceLaunchTimeoutError(undefined, deadline.timeoutMs) + } else { + await previous + } + return await operation() + } finally { + release() + } } findReadyInstanceIdByBinary(binaryPath: string): string | undefined { @@ -171,16 +267,13 @@ export class WorkspaceManager { })?.id } - private findReadyWorkspaceByIdentity( - identityKey: string, - includeRestoreOwned: boolean, - ): WorkspaceDescriptor | undefined { + private findReadyWorkspaceByIdentity(identityKey: string): WorkspaceDescriptor | undefined { for (const record of this.workspaces.values()) { const state = record[WORKSPACE_STATE] if ( state.published && !state.abortController.signal.aborted - && (includeRestoreOwned || !record.requestId) + && !state.deletePromise && record.status === "ready" && record.identityKey === identityKey ) { @@ -247,59 +340,128 @@ export class WorkspaceManager { const launchTimeoutMs = Math.max(1, this.options.launchTimeoutMs ?? DEFAULT_LAUNCH_TIMEOUT_MS) const launchDeadlineAt = Date.now() + launchTimeoutMs try { - const { workspacePath, identityKey } = await this.withLaunchDeadline( + let { workspacePath, identityKey } = await this.withLaunchDeadline( resolveWorkspaceIdentity(folder, this.options.rootDir), undefined, launchDeadlineAt, launchTimeoutMs, ) - if (options.requestId && this.cancelledCreationRequests.has(options.requestId)) { - throw new Error(`Workspace creation request ${options.requestId} was cancelled`) - } - if (this.shuttingDown) { - throw new Error("Workspace manager is shutting down") - } - if (options.forceNew) { - const ownership = this.createOwnership(options.requestId) - const record = this.reserveWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt) - const result = await this.startCreation(record, options, launchDeadlineAt, launchTimeoutMs) - return this.finishCreation(result, options.requestId, ownership) - } - const existing = this.findReadyWorkspaceByIdentity(identityKey, Boolean(options.requestId)) - if (existing) { - this.options.logger.info({ workspaceId: existing.id, folder: workspacePath }, "Reusing existing workspace") - const record = this.workspaces.get(existing.id) - if (options.requestId && record) { - if (!record.ownership.has(options.requestId)) record.ownership.set(options.requestId, "active") - this.syncOwnership(record) - return this.finishCreation({ workspace: existing, created: false }, options.requestId, record.ownership) + const pathKey = normalizeWorkspaceIdentityPath(workspacePath) + const finish = await this.withWorkspacePathKeyLease(pathKey, async () => { + const refreshed = await resolveWorkspaceIdentity(workspacePath, this.options.rootDir) + if (normalizeWorkspaceIdentityPath(refreshed.workspacePath) !== pathKey) { + throw new Error("Workspace path changed while waiting for another operation") } - return { workspace: existing, created: false } - } - const pending = this.pendingWorkspaceCreations.get(identityKey) - if (pending) { - const state = pending[WORKSPACE_STATE] - const owner = options.requestId ?? ORDINARY_CREATION_OWNER - if (!pending.ownership.has(owner)) pending.ownership.set(owner, "active") - this.syncOwnership(pending) - const result = await state.creation! - return this.finishCreation({ workspace: result.workspace, created: false }, options.requestId, pending.ownership) - } - const ownership = this.createOwnership(options.requestId) - const record = this.reserveWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt) - const creation = this.startCreation(record, options, launchDeadlineAt, launchTimeoutMs) - this.pendingWorkspaceCreations.set(identityKey, record) - try { - return this.finishCreation(await creation, options.requestId, ownership) - } finally { - if (this.pendingWorkspaceCreations.get(identityKey) === record) { - this.pendingWorkspaceCreations.delete(identityKey) + workspacePath = refreshed.workspacePath + identityKey = refreshed.identityKey + if (options.requestId && this.cancelledCreationRequests.has(options.requestId)) { + throw new Error(`Workspace creation request ${options.requestId} was cancelled`) } - } + if (this.shuttingDown) { + throw new Error("Workspace manager is shutting down") + } + const pathRecords = Array.from(this.workspaces.values()).filter((record) => + normalizeWorkspaceIdentityPath(record.path) === pathKey) + if (pathRecords.some((record) => record[WORKSPACE_STATE].deletePromise)) { + throw new Error("Workspace deletion is pending; wait for it to finish before relaunch") + } + if (pathRecords.some((record) => record[WORKSPACE_STATE].cleanupBlocked)) { + throw new Error("Workspace cleanup is incomplete and must be retried before relaunch") + } + if (options.lineageId) { + const lineageRecord = Array.from(this.workspaces.values()).find((record) => + record.lineageId === options.lineageId + && (record.status === "starting" || record.status === "ready") + && !record[WORKSPACE_STATE].deletePromise + && !record[WORKSPACE_STATE].abortController.signal.aborted) + if (lineageRecord) { + if (lineageRecord.identityKey !== identityKey) { + throw new Error("Workspace lineage belongs to a different workspace") + } + const owner = options.requestId ?? ORDINARY_CREATION_OWNER + if (!lineageRecord.ownership.has(owner)) lineageRecord.ownership.set(owner, "active") + this.syncOwnership(lineageRecord) + return async () => { + const workspace = lineageRecord[WORKSPACE_STATE].creation + ? (await lineageRecord[WORKSPACE_STATE].creation).workspace + : lineageRecord + return this.finishCreation({ workspace, created: false }, options.requestId, lineageRecord.ownership) + } + } + } + if (options.forceNew || options.lineageId) { + const ownership = this.createOwnership(options.requestId) + const record = await this.reserveLeasedWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt) + const creation = this.startCreation(record, options, launchDeadlineAt, launchTimeoutMs) + return async () => this.finishCreation(await creation, options.requestId, ownership) + } + const existing = this.findReadyWorkspaceByIdentity(identityKey) + if (existing) { + this.options.logger.info({ workspaceId: existing.id, folder: workspacePath }, "Reusing existing workspace") + const record = this.workspaces.get(existing.id) + if (!options.requestId && record && !record.ownership.has(ORDINARY_CREATION_OWNER)) { + record.ownership.set(ORDINARY_CREATION_OWNER, "active") + this.syncOwnership(record) + } + if (options.requestId && record) { + if (!record.ownership.has(options.requestId)) record.ownership.set(options.requestId, "active") + this.syncOwnership(record) + return async () => this.finishCreation({ workspace: existing, created: false }, options.requestId, record.ownership) + } + return async () => ({ workspace: existing, created: false }) + } + const pending = this.pendingWorkspaceCreations.get(identityKey) + if ( + pending + && !pending[WORKSPACE_STATE].deletePromise + && !pending[WORKSPACE_STATE].abortController.signal.aborted + && (pending.status === "starting" || pending.status === "ready") + ) { + const state = pending[WORKSPACE_STATE] + const owner = options.requestId ?? ORDINARY_CREATION_OWNER + if (!pending.ownership.has(owner)) pending.ownership.set(owner, "active") + this.syncOwnership(pending) + return async () => { + const result = await state.creation! + return this.finishCreation({ workspace: result.workspace, created: false }, options.requestId, pending.ownership) + } + } + const ownership = this.createOwnership(options.requestId) + const record = await this.reserveLeasedWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt) + const creation = this.startCreation(record, options, launchDeadlineAt, launchTimeoutMs) + this.pendingWorkspaceCreations.set(identityKey, record) + return async () => { + try { + return this.finishCreation(await creation, options.requestId, ownership) + } finally { + if (this.pendingWorkspaceCreations.get(identityKey) === record) { + this.pendingWorkspaceCreations.delete(identityKey) + } + } + } + }, { at: launchDeadlineAt, timeoutMs: launchTimeoutMs }) + return await finish() } finally { if (options.requestId) this.cancelledCreationRequests.delete(options.requestId) } } + private async reserveLeasedWorkspace( + workspacePath: string, + identityKey: string, + name: string | undefined, + options: WorkspaceCreateOptions, + ownership: WorkspaceCreationOwnership, + launchDeadlineAt: number, + ): Promise { + const processLease = await this.processLeases?.acquire(workspacePath) + if (this.processLeases && !processLease) throw new WorkspacePathOwnedError(workspacePath) + try { + return this.reserveWorkspace(workspacePath, identityKey, name, options, ownership, launchDeadlineAt, processLease) + } catch (error) { + await processLease?.release() + throw error + } + } private reserveWorkspace( workspacePath: string, identityKey: string, @@ -307,6 +469,7 @@ export class WorkspaceManager { options: WorkspaceCreateOptions, ownership: WorkspaceCreationOwnership, launchDeadlineAt: number, + processLease?: WorkspaceProcessLease, ): WorkspaceRecord { const id = randomUUID() const binary = this.options.binaryResolver.resolve(options.binaryPath) @@ -320,6 +483,7 @@ export class WorkspaceManager { const record = { id, + lineageId: options.lineageId ?? randomUUID(), requestId: options.requestId, path: workspacePath, name, @@ -334,10 +498,11 @@ export class WorkspaceManager { Object.defineProperties(record, { identityKey: { value: identityKey }, ownership: { value: ownership }, - [WORKSPACE_STATE]: { value: { abortController: new AbortController(), published: false, stoppedPublished: false } }, + [WORKSPACE_STATE]: { value: { abortController: new AbortController(), published: false, stoppedPublished: false, processLease } }, }) this.workspaces.set(id, record) + processLease?.onLost(() => this.handleProcessLeaseLost(id, record)) if (options.requestId && this.cancelledCreationRequests.has(options.requestId)) { record[WORKSPACE_STATE].abortController.abort(new WorkspaceLaunchCancelledError(id)) } @@ -411,6 +576,8 @@ export class WorkspaceManager { throw new Error("Failed to build OpenCode auth header") } this.opencodeAuth.set(id, { username: opencodeUsername, password: opencodePassword, authorization }) + const callbackToken = generateCodeNomadCallbackToken() + this.pluginCallbackAuth.set(id, buildCodeNomadCallbackAuthorizationHeader(callbackToken)) const environment = { ...userEnvironment, @@ -418,6 +585,7 @@ export class WorkspaceManager { OPENCODE_EXPERIMENTAL_WORKSPACES: "true", CODENOMAD_INSTANCE_ID: id, CODENOMAD_BASE_URL: serverBaseUrl, + [CODENOMAD_CALLBACK_TOKEN_ENV]: callbackToken, ...(this.options.nodeExtraCaCertsPath ? { NODE_EXTRA_CA_CERTS: this.options.nodeExtraCaCertsPath } : {}), [OPENCODE_SERVER_BASE_URL_ENV]: `${normalizedServerBaseUrl}${proxyPath}`, [OPENCODE_SERVER_USERNAME_ENV]: opencodeUsername, @@ -425,13 +593,16 @@ export class WorkspaceManager { } const logLevel = (serverConfig as any)?.logLevel + const cleanupToken = await state.processLease?.prepareLaunch() const { pid, port, exitPromise, getLastOutput } = await this.runtime.launch({ workspaceId: id, folder: workspacePath, binaryPath: resolvedBinaryPath, environment, logLevel, + cleanupToken, signal: state.abortController.signal, + persistProcessIdentities: (identities) => state.processLease?.setProcessIdentities(identities) ?? Promise.resolve(), onExit: (info) => this.handleProcessExit(info.workspaceId, info), }) record.pid = pid @@ -465,17 +636,18 @@ export class WorkspaceManager { stopFailure = stopError }) if (!stopFailure) { - this.removeRecord(id, record, state.published) + await this.removeRecord(id, record, state.published) throw launchFailure } - if (!state.published) { - throw stopFailure - } + state.cleanupBlocked = true record.status = "error" record.error = stopFailure instanceof Error ? `Workspace startup failed and its process could not be stopped: ${stopFailure.message}` : launchFailure instanceof Error ? launchFailure.message : String(launchFailure) record.updatedAt = new Date().toISOString() + if (!state.published) { + throw stopFailure + } if (this.workspaces.get(id) === record && state.published) { this.options.eventBus.publish({ type: "workspace.error", workspace: record }) } @@ -488,19 +660,27 @@ export class WorkspaceManager { const record = this.workspaces.get(id) if (!record) return Promise.resolve(undefined) const state = record[WORKSPACE_STATE] - if (!state.abortController.signal.aborted) { - state.abortController.abort(new WorkspaceLaunchCancelledError(id)) - } - const pending = this.pendingWorkspaceCreations.get(record.identityKey) - if (pending === record) { - this.pendingWorkspaceCreations.delete(record.identityKey) - } if (!state.deletePromise) { - let deletePromise!: Promise - deletePromise = this.cleanupDeletedWorkspace(id, record).catch((error) => { - if (state.deletePromise === deletePromise) state.deletePromise = undefined - throw error - }) + let cleanupStarted = false + const cleanup = () => { + cleanupStarted = true + if (!state.abortController.signal.aborted) { + state.abortController.abort(new WorkspaceLaunchCancelledError(id)) + } + if (this.pendingWorkspaceCreations.get(record.identityKey) === record) { + this.pendingWorkspaceCreations.delete(record.identityKey) + } + return this.cleanupDeletedWorkspace(id, record) + } + const deletePromise = Promise.resolve() + .then(() => this.shuttingDown || !this.deletionGuard + ? cleanup() + : this.deletionGuard(record, cleanup)) + .catch((error) => { + if (cleanupStarted && this.workspaces.get(id) === record) state.cleanupBlocked = true + if (state.deletePromise === deletePromise) state.deletePromise = undefined + throw error + }) state.deletePromise = deletePromise } return state.deletePromise @@ -630,18 +810,38 @@ export class WorkspaceManager { await immediateStop await this.runtime.stop(id) - this.removeRecord(id, record, true) + await this.removeRecord(id, record, true) return record } - private removeRecord(id: string, record: WorkspaceRecord, publishStopped: boolean): void { + private async removeRecord(id: string, record: WorkspaceRecord, publishStopped: boolean): Promise { if (this.workspaces.get(id) !== record) return + const processLease = record[WORKSPACE_STATE].processLease + await processLease?.release() + record[WORKSPACE_STATE].processLease = undefined this.workspaces.delete(id) this.opencodeAuth.delete(id) + this.pluginCallbackAuth.delete(id) clearWorkspaceSearchCache(record.path) if (publishStopped) this.publishStopped(record, "deleted") } + private handleProcessLeaseLost(id: string, record: WorkspaceRecord): void { + if (this.workspaces.get(id) !== record) return + const state = record[WORKSPACE_STATE] + if (state.leaseLost) return + state.leaseLost = true + state.cleanupBlocked = true + record.status = "error" + record.error = "Workspace process lease was lost; the workspace is being stopped" + record.updatedAt = new Date().toISOString() + state.abortController.abort(new WorkspacePathOwnedError(record.path)) + if (state.published) this.options.eventBus.publish({ type: "workspace.error", workspace: record }) + void this.runtime.stop(id) + .then(() => this.removeRecord(id, record, state.published)) + .catch((error) => this.options.logger.error({ workspaceId: id, err: error }, "Workspace lease-loss cleanup remains retryable")) + } + private publishStopped(record: WorkspaceRecord, reason: "deleted" | "stopped" = "stopped"): void { const state = record[WORKSPACE_STATE] if (!state.published || state.stoppedPublished) return @@ -876,6 +1076,7 @@ export class WorkspaceManager { const workspace = record this.opencodeAuth.delete(workspaceId) + this.pluginCallbackAuth.delete(workspaceId) this.options.logger.info({ workspaceId, ...info }, "Workspace process exited") diff --git a/packages/server/src/workspaces/opencode-auth.test.ts b/packages/server/src/workspaces/opencode-auth.test.ts index e4a13a4d5..ea766592f 100644 --- a/packages/server/src/workspaces/opencode-auth.test.ts +++ b/packages/server/src/workspaces/opencode-auth.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" -import { resolveOpencodeServerAuth } from "./opencode-auth" +import { buildCodeNomadCallbackAuthorizationHeader, generateCodeNomadCallbackToken, resolveOpencodeServerAuth } from "./opencode-auth" describe("resolveOpencodeServerAuth", () => { it("uses configured OpenCode auth from workspace environment", () => { @@ -39,3 +39,9 @@ describe("resolveOpencodeServerAuth", () => { assert.deepEqual(auth, { username: "codenomad", password: "generated" }) }) }) + +it("builds a bearer callback capability", () => { + const token = generateCodeNomadCallbackToken() + assert.match(token, /^[A-Za-z0-9_-]{43}$/) + assert.equal(buildCodeNomadCallbackAuthorizationHeader(token), `Bearer ${token}`) +}) diff --git a/packages/server/src/workspaces/opencode-auth.ts b/packages/server/src/workspaces/opencode-auth.ts index 55daeed7b..c2ad332bc 100644 --- a/packages/server/src/workspaces/opencode-auth.ts +++ b/packages/server/src/workspaces/opencode-auth.ts @@ -3,6 +3,7 @@ import crypto from "node:crypto" export const OPENCODE_SERVER_USERNAME_ENV = "OPENCODE_SERVER_USERNAME" as const export const OPENCODE_SERVER_PASSWORD_ENV = "OPENCODE_SERVER_PASSWORD" as const export const OPENCODE_SERVER_BASE_URL_ENV = "OPENCODE_SERVER_BASE_URL" as const +export const CODENOMAD_CALLBACK_TOKEN_ENV = "CODENOMAD_CALLBACK_TOKEN" as const export const DEFAULT_OPENCODE_USERNAME = "codenomad" as const @@ -10,6 +11,10 @@ export function generateOpencodeServerPassword(): string { return crypto.randomBytes(32).toString("base64url") } +export function generateCodeNomadCallbackToken(): string { + return crypto.randomBytes(32).toString("base64url") +} + function readConfiguredValue(key: string, ...sources: Array | undefined>): string | undefined { for (const source of sources) { const value = source?.[key] @@ -47,3 +52,7 @@ export function buildOpencodeBasicAuthHeader(params: { username?: string; passwo const token = Buffer.from(`${username}:${password}`, "utf8").toString("base64") return `Basic ${token}` } + +export function buildCodeNomadCallbackAuthorizationHeader(token: string): string { + return `Bearer ${token}` +} diff --git a/packages/server/src/workspaces/process-identity.darwin.test.ts b/packages/server/src/workspaces/process-identity.darwin.test.ts index f657474f1..c2404e0ee 100644 --- a/packages/server/src/workspaces/process-identity.darwin.test.ts +++ b/packages/server/src/workspaces/process-identity.darwin.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict" import { spawn, spawnSync } from "node:child_process" +import { randomUUID } from "node:crypto" import { once } from "node:events" import { setTimeout as delay } from "node:timers/promises" import { it } from "node:test" @@ -7,6 +8,8 @@ import { it } from "node:test" import { LAUNCH_CLEANUP_TOKEN_ENV, probePosixProcesses, + probeLaunchCleanupToken, + signalLaunchCleanupToken, signalOwnedPosixProcessGroup, signalPosixProcesses, } from "./process-identity" @@ -63,6 +66,42 @@ it("uses real Darwin ps identities to stop an owned detached process group", dar } }) +it("finds and stops cleanup-token descendants that escape the original Darwin process group", darwinOnly, async () => { + const cleanupToken = `darwin-escaped-${randomUUID()}` + const leader = spawn(process.execPath, ["-e", ` + const { spawn } = require("node:child_process") + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore", env: process.env }) + process.stdout.write(String(child.pid) + "\\n") + setInterval(() => {}, 1000) + `], { + detached: true, + stdio: ["ignore", "pipe", "ignore"], + env: { ...process.env, [LAUNCH_CLEANUP_TOKEN_ENV]: cleanupToken }, + }) + assert.ok(leader.pid) + const [chunk] = await once(leader.stdout!, "data") + const escapedPid = Number.parseInt(String(chunk), 10) + assert.ok(escapedPid > 0) + + try { + process.kill(-leader.pid, "SIGTERM") + if (leader.exitCode === null) await once(leader, "exit") + const escaped = probeLaunchCleanupToken(spawnSync, cleanupToken, 1_000, undefined, "darwin") + assert.equal(escaped.ok && escaped.processes.has(escapedPid), true) + const signaled = signalLaunchCleanupToken(spawnSync, cleanupToken, "SIGTERM", 1_000, undefined, "darwin") + assert.equal(signaled.ok && signaled.targets.some(({ pid }) => pid === escapedPid), true) + for (let attempt = 0; attempt < 20; attempt += 1) { + const remaining = probeLaunchCleanupToken(spawnSync, cleanupToken, 1_000, undefined, "darwin") + if (remaining.ok && remaining.processes.size === 0) return + await delay(50) + } + assert.fail("escaped Darwin cleanup-token descendant remained alive") + } finally { + try { process.kill(escapedPid, "SIGKILL") } catch {} + try { process.kill(-leader.pid, "SIGKILL") } catch {} + } +}) + it("uses a retained real Darwin identity anchor after the group leader exits", darwinOnly, async () => { const cleanupToken = "darwin-integration-cleanup-token" const leader = await spawnDetachedGroup(cleanupToken) diff --git a/packages/server/src/workspaces/process-identity.ts b/packages/server/src/workspaces/process-identity.ts index 2d9b2f070..368299b3e 100644 --- a/packages/server/src/workspaces/process-identity.ts +++ b/packages/server/src/workspaces/process-identity.ts @@ -192,6 +192,27 @@ fi printf 'CODENOMAD_RESULT|%s||%s\n' "$matched" "$signal_sent" ` +const POSIX_TOKEN_SCRIPT = String.raw`${POSIX_IDENTITY_FUNCTIONS} +cleanup_token=$1; requested_signal=$2; signal_sent=0; passes=1; test -n "$requested_signal" && passes=3 +emit_token_identity() { + printf '%s|%s|%s|%s|' "$1" "$current_pid" "$current_ppid" "$current_group" + encode "$current_start"; printf '|'; encode "$current_command"; printf '\n' +} +pass=0 +while test "$pass" -lt "$passes"; do + pass=$((pass + 1)) + for current_pid in $(ps -axo pid= 2>/dev/null); do + if has_cleanup_token "$current_pid" && read_identity "$current_pid"; then + test -n "$requested_signal" && prefix=CODENOMAD_TARGET_B64 || prefix=CODENOMAD_PROCESS_B64 + emit_token_identity "$prefix" + if test -n "$requested_signal" && has_cleanup_token "$current_pid" && read_identity "$current_pid" && kill "-$requested_signal" "$current_pid" 2>/dev/null; then signal_sent=1; fi + fi + done +done +if test -n "$requested_signal"; then printf 'CODENOMAD_RESULT|%s\n' "$signal_sent"; fi +exit 0 +` + const commandError = (result: SpawnSyncReturns): string => result.error?.message || String(result.stderr ?? result.stdout ?? "").trim() || `exit code ${result.status}` @@ -347,6 +368,13 @@ function runLinuxScript(spawnCommand: SpawnCommand, script: string, args: string : spawnCommand("sh", ["-c", script, label, ...args], { encoding: "utf8", timeout: timeoutMs }) } +function runPortablePosixTokenScript(spawnCommand: SpawnCommand, token: string, requestedSignal: string, + timeoutMs: number): SpawnSyncReturns { + return spawnCommand("sh", ["-c", POSIX_TOKEN_SCRIPT, "codenomad-token-cleanup", token, requestedSignal], { + encoding: "utf8", timeout: timeoutMs, + }) +} + const redactToken = (value: string, token: string): string => value.split(token).join("[REDACTED]") function shellGuardArgs(request: GuardedSignalRequest, linux: boolean): string[] { @@ -511,17 +539,25 @@ export function signalWindowsProcesses(spawnCommand: SpawnCommand, request: Guar } export function probeLaunchCleanupToken(spawnCommand: SpawnCommand, token: string, - timeoutMs: number, distro?: string): ProcessSnapshot { + timeoutMs: number, distro?: string, platform: NodeJS.Platform = "linux"): ProcessSnapshot { + if (!distro && platform === "win32") { + return { ok: false, error: "Windows cannot inspect another process environment; use persisted launch-tree identities" } + } + const portablePosix = !distro && platform !== "linux" return querySnapshot( - () => runLinuxScript( - spawnCommand, - LINUX_TOKEN_SCRIPT, - [LAUNCH_CLEANUP_TOKEN_ENV, token, ""], - timeoutMs, - "codenomad-token-cleanup", - distro, - ), - (output) => parsePrefixedSnapshot(output, "CODENOMAD_PROCESS|"), + () => portablePosix + ? runPortablePosixTokenScript(spawnCommand, token, "", timeoutMs) + : runLinuxScript( + spawnCommand, + LINUX_TOKEN_SCRIPT, + [LAUNCH_CLEANUP_TOKEN_ENV, token, ""], + timeoutMs, + "codenomad-token-cleanup", + distro, + ), + (output) => portablePosix + ? parseBase64Snapshot(output, "CODENOMAD_PROCESS_B64|") + : parsePrefixedSnapshot(output, "CODENOMAD_PROCESS|"), { allowEmpty: true, malformedError: "launch cleanup probe returned malformed or unexpected output", @@ -531,27 +567,30 @@ export function probeLaunchCleanupToken(spawnCommand: SpawnCommand, token: strin } export function signalLaunchCleanupToken(spawnCommand: SpawnCommand, token: string, - signal: NodeJS.Signals, timeoutMs: number, distro?: string): TokenSignalResult { + signal: NodeJS.Signals, timeoutMs: number, distro?: string, platform: NodeJS.Platform = "linux"): TokenSignalResult { const failed = (error: string): TokenSignalResult => ({ ok: false, signalSent: false, targets: [], error }) try { - const result = runLinuxScript( - spawnCommand, - LINUX_TOKEN_SCRIPT, - [LAUNCH_CLEANUP_TOKEN_ENV, token, signalName(signal)], - timeoutMs, - "codenomad-token-cleanup", - distro, - ) + const portablePosix = !distro && platform !== "linux" + const result = portablePosix + ? runPortablePosixTokenScript(spawnCommand, token, signalName(signal), timeoutMs) + : runLinuxScript( + spawnCommand, + LINUX_TOKEN_SCRIPT, + [LAUNCH_CLEANUP_TOKEN_ENV, token, signalName(signal)], + timeoutMs, + "codenomad-token-cleanup", + distro, + ) if (result.status !== 0) return failed(redactToken(commandError(result), token)) const lines = String(result.stdout ?? "").split(/\r?\n/).filter(Boolean) const resultLines = lines.filter((line) => line.startsWith("CODENOMAD_RESULT|")) if (resultLines.length !== 1 || !/^CODENOMAD_RESULT\|[01]$/.test(resultLines[0] ?? "")) { return failed("launch cleanup signal returned no valid structured result") } - const targets = parsePrefixedSnapshot( - lines.filter((line) => !line.startsWith("CODENOMAD_RESULT|")).join("\n"), - "CODENOMAD_TARGET|", - ) + const targetOutput = lines.filter((line) => !line.startsWith("CODENOMAD_RESULT|")).join("\n") + const targets = portablePosix + ? parseBase64Snapshot(targetOutput, "CODENOMAD_TARGET_B64|") + : parsePrefixedSnapshot(targetOutput, "CODENOMAD_TARGET|") return targets ? { ok: true, signalSent: resultLines[0]!.endsWith("1"), targets: Array.from(targets.values()) } : failed("launch cleanup signal returned malformed or unexpected output") diff --git a/packages/server/src/workspaces/process-lease.test.ts b/packages/server/src/workspaces/process-lease.test.ts new file mode 100644 index 000000000..eab86e5d0 --- /dev/null +++ b/packages/server/src/workspaces/process-lease.test.ts @@ -0,0 +1,438 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { writeFileSync } from "node:fs" +import { mkdir, mkdtemp, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { test } from "node:test" + +import { WorkspaceProcessLeaseRegistry } from "./process-lease" + +test("a stale lease is reclaimed without an old owner deleting its replacement", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-lease-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const stale = new WorkspaceProcessLeaseRegistry({ + directory, + managerToken: "stale-manager", + pid: 101, + hostname: "same-host", + isPidAlive: () => false, + }) + const replacement = new WorkspaceProcessLeaseRegistry({ + directory, + managerToken: "replacement-manager", + pid: 202, + hostname: "same-host", + isPidAlive: (pid) => pid === 202, + }) + + const staleLease = await stale.acquire(workspacePath) + assert.ok(staleLease) + const replacementLease = await replacement.acquire(workspacePath) + assert.ok(replacementLease) + + await staleLease.release() + const blocked = new WorkspaceProcessLeaseRegistry({ + directory, + managerToken: "blocked-manager", + isPidAlive: (pid) => pid === 202, + }) + assert.equal(await blocked.acquire(workspacePath), undefined) + await replacementLease.release() + const successorLease = await blocked.acquire(workspacePath) + assert.ok(successorLease) + await successorLease.release() +}) + +test("a live detached process identity keeps a same-host lease valid", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-identity-lease-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + let processAlive = true + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: () => false, + isProcessIdentityAlive: () => processAlive, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "immutable-start" }]) + + assert.equal(await contender.acquire(workspacePath), undefined) + processAlive = false + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await lease.release() + await replacement.release() +}) + +test("an inconclusive detached process probe fails closed", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-identity-unknown-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: () => false, + isProcessIdentityAlive: () => undefined, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "immutable-start" }]) + + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() +}) + +test("a same-host live server blocks takeover without relying on heartbeat age", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-live-server-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ directory, managerToken: "owner", pid: 101, hostname: "same-host" }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: (pid) => pid === 101, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() +}) + +test("owner replacement reports lease loss", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-lease-loss-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", heartbeatMs: 10, staleMs: 20, + isPidAlive: () => false, + }) + const replacement = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "replacement", pid: 202, hostname: "same-host", heartbeatMs: 60_000, staleMs: 20, + isPidAlive: (pid) => pid === 202, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + const lost = new Promise((resolve) => lease.onLost(resolve)) + const successor = await replacement.acquire(workspacePath) + assert.ok(successor) + await lost + await lease.release().catch(() => undefined) + await successor.release() +}) + +test("a pre-spawn cleanup token closes the process identity publication gap", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-launch-anchor-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + let launchAlive = true + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: () => false, + platform: "linux", isLaunchTokenAlive: () => launchAlive, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + const token = await lease.prepareLaunch() + assert.ok(token) + assert.equal(await contender.acquire(workspacePath), undefined) + + launchAlive = false + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await lease.release() + await replacement.release() +}) + +test("an unknown launch token is checked before process identity discovery", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-launch-first-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: () => false, + isLaunchTokenAlive: () => undefined, + isProcessIdentityAlive: () => { throw new Error("process discovery failed") }, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "immutable-start" }]) + + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() +}) + +test("an inconclusive Windows token probe falls through to immutable identities", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-win32-token-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + let processAlive = true + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform: "win32", + isPidAlive: () => false, isLaunchTokenAlive: () => undefined, isProcessIdentityAlive: () => processAlive, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "windows-creation-ticks" }]) + + assert.equal(await contender.acquire(workspacePath), undefined) + processAlive = false + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await lease.release() + await replacement.release() +}) + +for (const platform of ["linux", "win32"] as const) { + test(`a complete legacy ${platform} lease blocks while live and reclaims when dead`, async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-legacy-complete-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + let processAlive = true + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform, + isPidAlive: () => false, isLaunchTokenAlive: () => platform === "linux" ? false : undefined, + isProcessIdentityAlive: () => processAlive, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + const [key] = await readdir(directory) + const leaseDirectory = path.join(directory, key!) + const ownerRecord = JSON.parse(await readFile(path.join(leaseDirectory, "owner", "owner.json"), "utf8")) + const serialized = JSON.stringify({ pid: 303, parentPid: 1, groupId: 303, startTime: "legacy-start" }) + const digest = createHash("sha256").update(serialized).digest("hex") + await writeFile(path.join(leaseDirectory, `launch.${ownerRecord.leaseToken}.json`), JSON.stringify({ token: "legacy-token" })) + await writeFile(path.join(leaseDirectory, `process.${ownerRecord.leaseToken}.${digest}.json`), serialized) + + assert.equal(await contender.acquire(workspacePath), undefined) + processAlive = false + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await lease.release() + await replacement.release() + }) + + for (const legacyState of ["partial", "malformed"] as const) { + test(`a ${legacyState} legacy ${platform} lease fails closed`, async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-legacy-incomplete-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform, + isPidAlive: () => false, isLaunchTokenAlive: () => false, isProcessIdentityAlive: () => false, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + const [key] = await readdir(directory) + const leaseDirectory = path.join(directory, key!) + const ownerRecord = JSON.parse(await readFile(path.join(leaseDirectory, "owner", "owner.json"), "utf8")) + await writeFile(path.join(leaseDirectory, `launch.${ownerRecord.leaseToken}.json`), JSON.stringify({ token: "legacy-token" })) + if (legacyState === "malformed") { + await writeFile(path.join(leaseDirectory, `process.${ownerRecord.leaseToken}.invalid.json`), "{") + } + + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() + }) + } +} + +test("a torn Windows identity generation remains an incomplete takeover fence", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-win32-torn-generation-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform: "win32", + isPidAlive: () => false, isLaunchTokenAlive: () => false, isProcessIdentityAlive: () => false, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + const [key] = await readdir(directory) + const leaseDirectory = path.join(directory, key!) + const ownerRecord = JSON.parse(await readFile(path.join(leaseDirectory, "owner", "owner.json"), "utf8")) + const launchFile = (await readdir(leaseDirectory)).find((entry) => entry.startsWith(`launch.${ownerRecord.leaseToken}.`)) + assert.ok(launchFile) + await writeFile(path.join(leaseDirectory, launchFile), JSON.stringify({ + version: 1, + generation: "partial-generation", + complete: false, + identities: [{ pid: 303, parentPid: 1, groupId: 303, startTime: "wrapper-only" }], + })) + + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() +}) + +test("one complete Windows launch cannot hide an incomplete sibling generation", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-win32-generations-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform: "win32", + isPidAlive: () => false, isLaunchTokenAlive: () => false, isProcessIdentityAlive: () => false, + }) + const first = await owner.acquire(workspacePath) + const second = await owner.acquire(workspacePath) + assert.ok(first && second) + await first.prepareLaunch() + await second.prepareLaunch() + await first.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "first" }]) + + assert.equal(await contender.acquire(workspacePath), undefined) + await second.setProcessIdentities([{ pid: 304, parentPid: 1, groupId: 304, startTime: "second" }]) + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await first.release() + await second.release() + await replacement.release() +}) + +for (const scenario of [ + { name: "a live direct Windows launch", persisted: [303], alive: [303], reclaim: false }, + { name: "a live Windows wrapper", persisted: [303, 304], alive: [303, 304], reclaim: false }, + { name: "a surviving Windows wrapper child", persisted: [303, 304], alive: [304], reclaim: false }, + { name: "a dead Windows wrapper tree", persisted: [303, 304], alive: [], reclaim: true }, +] as const) { + test(`${scenario.name} controls same-host lease recovery`, async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-win32-tree-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const alive = new Set(scenario.alive) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform: "win32", + isPidAlive: () => false, isLaunchTokenAlive: () => undefined, + isProcessIdentityAlive: (identity) => alive.has(identity.pid), + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities(scenario.persisted.map((pid) => ({ + pid, parentPid: pid === 303 ? 1 : 303, groupId: pid, startTime: `windows-${pid}`, + }))) + + const replacement = await contender.acquire(workspacePath) + assert.equal(Boolean(replacement), scenario.reclaim) + await lease.release() + await replacement?.release() + }) +} + +test("a stale torn launch token does not permanently wedge a dead owner", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-torn-launch-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", heartbeatMs: 60_000, isPidAlive: () => false, + }) + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", platform: "linux", staleMs: 20, isPidAlive: () => false, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + const [key] = await readdir(directory) + const leaseDirectory = path.join(directory, key!) + const ownerRecord = JSON.parse(await readFile(path.join(leaseDirectory, "owner", "owner.json"), "utf8")) + const launchFile = (await readdir(leaseDirectory)).find((entry) => entry.startsWith(`launch.${ownerRecord.leaseToken}.`)) + assert.ok(launchFile) + const launchPath = path.join(leaseDirectory, launchFile) + await writeFile(launchPath, "{", "utf8") + + assert.equal(await contender.acquire(workspacePath), undefined) + const old = new Date(Date.now() - 1_000) + await Promise.all([ + utimes(launchPath, old, old), + utimes(path.join(leaseDirectory, "owner", "heartbeat"), old, old), + ]) + const replacement = await contender.acquire(workspacePath) + assert.ok(replacement) + await lease.release() + await replacement.release() +}) + +test("foreign-host owners fail closed regardless of heartbeat age", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-foreign-owner-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ directory, hostname: "foreign-host", pid: 101, isPidAlive: () => false }) + const contender = new WorkspaceProcessLeaseRegistry({ directory, hostname: "local-host", pid: 202, isPidAlive: () => false }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + assert.equal(await contender.acquire(workspacePath), undefined) + await lease.release() +}) + +test("retirement CAS includes the observed heartbeat generation", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-heartbeat-cas-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const workspacePath = path.join(directory, "workspace") + const owner = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "owner", pid: 101, hostname: "same-host", isPidAlive: () => false, + }) + const lease = await owner.acquire(workspacePath) + assert.ok(lease) + await lease.prepareLaunch() + await lease.setProcessIdentities([{ pid: 303, parentPid: 1, groupId: 303, startTime: "immutable-start" }]) + const [key] = await readdir(directory) + const ownerDirectory = path.join(directory, key!, "owner") + const contender = new WorkspaceProcessLeaseRegistry({ + directory, managerToken: "contender", pid: 202, hostname: "same-host", isPidAlive: () => false, + isProcessIdentityAlive: () => { + writeFileSync(path.join(ownerDirectory, "heartbeat"), "new-generation") + return false + }, + }) + + assert.equal(await contender.acquire(workspacePath), undefined) + assert.equal(JSON.parse(await readFile(path.join(ownerDirectory, "owner.json"), "utf8")).managerToken, "owner") + await lease.release() +}) + +test("failed final retirement can be retried", async (t) => { + const directory = await mkdtemp(path.join(os.tmpdir(), "codenomad-process-lease-release-")) + t.after(() => rm(directory, { recursive: true, force: true })) + const registry = new WorkspaceProcessLeaseRegistry({ directory, isPidAlive: () => true }) + const lease = await registry.acquire(path.join(directory, "workspace")) + assert.ok(lease) + const [key] = await readdir(directory) + const leaseDirectory = path.join(directory, key!) + const owner = JSON.parse(await readFile(path.join(leaseDirectory, "owner", "owner.json"), "utf8")) + const tombstone = path.join(leaseDirectory, `retired.${owner.leaseToken}`) + await mkdir(tombstone) + + await assert.rejects(lease.release(), /release can be retried/) + await rm(tombstone, { recursive: true }) + const retry = await registry.acquire(path.join(directory, "workspace")) + assert.ok(retry) + await assert.doesNotReject(retry.release()) +}) diff --git a/packages/server/src/workspaces/process-lease.ts b/packages/server/src/workspaces/process-lease.ts new file mode 100644 index 000000000..29302cc02 --- /dev/null +++ b/packages/server/src/workspaces/process-lease.ts @@ -0,0 +1,526 @@ +import { createHash, randomUUID } from "node:crypto" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" + +import { normalizeWorkspaceIdentityPath } from "./workspace-identity" +import { + probeLaunchCleanupToken, + probePosixProcesses, + probeWindowsProcesses, + sameProcess, + type ProcessIdentity, +} from "./process-identity" + +const OWNER_FILE = "owner.json" +const DEFAULT_HEARTBEAT_MS = 5_000 +const DEFAULT_STALE_MS = 15_000 + +interface LeaseOwner { + version: 1 + managerToken: string + leaseToken: string + pid: number + hostname: string + processStart?: string + workspacePath: string +} + +export interface WorkspaceProcessLease { + release(): Promise + prepareLaunch(): Promise + setProcessIdentities(identities: ProcessIdentity[]): Promise + onLost(callback: () => void): void +} + +export interface WorkspaceProcessLeaseRegistryOptions { + directory: string + heartbeatMs?: number + staleMs?: number + managerToken?: string + pid?: number + hostname?: string + processStart?: string + isPidAlive?: (pid: number) => boolean + isProcessIdentityAlive?: (identity: ProcessIdentity) => boolean | undefined + isLaunchTokenAlive?: (token: string) => boolean | undefined + platform?: NodeJS.Platform +} + +interface ObservedOwner { + owner: LeaseOwner + serialized: string + heartbeat: string +} + +interface HeldLease { + count: number + directory: string + owner: LeaseOwner + heartbeat: NodeJS.Timeout + lost: boolean + releaseFailed: boolean + onLost: Set<() => void> +} + +interface LaunchGeneration { + version: 1 + generation: string + token: string + complete: boolean + identities?: ProcessIdentity[] +} + +export class WorkspaceProcessLeaseRegistry { + private readonly managerToken: string + private readonly pid: number + private readonly hostname: string + private readonly heartbeatMs: number + private readonly staleMs: number + private readonly held = new Map() + private processStart: string | undefined + + constructor(private readonly options: WorkspaceProcessLeaseRegistryOptions) { + this.managerToken = options.managerToken ?? randomUUID() + this.pid = options.pid ?? process.pid + this.hostname = options.hostname ?? os.hostname() + this.processStart = options.processStart + this.heartbeatMs = Math.max(10, options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS) + this.staleMs = Math.max(10, options.staleMs ?? DEFAULT_STALE_MS) + } + + async acquire(workspacePath: string): Promise { + const canonicalPath = normalizeWorkspaceIdentityPath(workspacePath) + const key = createHash("sha256").update(canonicalPath).digest("hex") + const existing = this.held.get(key) + if (existing) { + if (existing.releaseFailed) existing.releaseFailed = false + else existing.count += 1 + return this.handle(key, existing.owner.leaseToken) + } + + const directory = path.join(this.options.directory, key) + await fs.mkdir(directory, { recursive: true }) + this.processStart ??= await readProcessStart(this.pid) + + for (let attempt = 0; attempt < 3; attempt += 1) { + const owner: LeaseOwner = { + version: 1, + managerToken: this.managerToken, + leaseToken: randomUUID(), + pid: this.pid, + hostname: this.hostname, + processStart: this.processStart, + workspacePath: canonicalPath, + } + if (await publishOwner(directory, owner)) { + const heartbeat = setInterval(() => void this.heartbeat(key, owner), this.heartbeatMs) + heartbeat.unref() + this.held.set(key, { count: 1, directory, owner, heartbeat, lost: false, releaseFailed: false, onLost: new Set() }) + return this.handle(key, owner.leaseToken) + } + + const observed = await readOwner(directory) + if (!observed || !await this.ownerIsStale(directory, observed.owner)) return undefined + if (!await retireOwner(directory, observed)) return undefined + } + return undefined + } + + private handle(key: string, leaseToken: string): WorkspaceProcessLease { + let released = false + let lostCallback: (() => void) | undefined + let launchGeneration: string | undefined + let launchToken: string | undefined + return { + release: async () => { + if (released) return + const held = this.held.get(key) + if (!held || held.owner.leaseToken !== leaseToken) return + if (held.count > 1) { + if (launchGeneration) await removeLaunchGeneration(held.directory, leaseToken, launchGeneration) + held.count -= 1 + released = true + if (lostCallback) held.onLost.delete(lostCallback) + return + } + try { + const observed = await readOwner(held.directory) + if (observed?.owner.leaseToken === leaseToken + && !await retireOwner(held.directory, observed)) { + throw new Error(`Workspace process lease ${leaseToken} could not be retired; release can be retried`) + } + } catch (error) { + held.releaseFailed = true + throw error + } + released = true + this.held.delete(key) + clearInterval(held.heartbeat) + if (lostCallback) held.onLost.delete(lostCallback) + }, + prepareLaunch: async () => { + const held = this.held.get(key) + if (!held || held.owner.leaseToken !== leaseToken || held.lost) throw new Error("Workspace process lease was lost") + if (launchToken) return launchToken + const generation = randomUUID() + const token = randomUUID() + await writeLaunchGeneration(held.directory, leaseToken, { version: 1, generation, token, complete: false }) + if ((await readOwner(held.directory))?.owner.leaseToken !== leaseToken) { + this.lose(key, held) + throw new Error("Workspace process lease was lost") + } + launchGeneration = generation + launchToken = token + return token + }, + setProcessIdentities: async (identities) => { + const held = this.held.get(key) + if (!held || held.owner.leaseToken !== leaseToken || held.lost) throw new Error("Workspace process lease was lost") + if (!launchGeneration || !launchToken) throw new Error("Workspace launch was not prepared") + await writeLaunchGeneration(held.directory, leaseToken, { + version: 1, generation: launchGeneration, token: launchToken, complete: true, identities, + }, true) + if ((await readOwner(held.directory))?.owner.leaseToken !== leaseToken) { + this.lose(key, held) + throw new Error("Workspace process lease was lost") + } + }, + onLost: (callback) => { + const held = this.held.get(key) + if (!held || held.owner.leaseToken !== leaseToken || held.lost) queueMicrotask(callback) + else { + if (lostCallback) held.onLost.delete(lostCallback) + lostCallback = callback + held.onLost.add(callback) + } + }, + } + } + + private async heartbeat(key: string, owner: LeaseOwner): Promise { + const held = this.held.get(key) + if (!held || held.owner.leaseToken !== owner.leaseToken || held.lost) return + try { + if (await heartbeatOwner(held.directory, owner)) return + } catch { + // Fail closed: durable ownership cannot be assumed after a heartbeat I/O failure. + } + this.lose(key, held) + } + + private lose(key: string, held: HeldLease): void { + if (this.held.get(key) !== held || held.lost) return + held.lost = true + clearInterval(held.heartbeat) + for (const callback of held.onLost) callback() + } + + private async ownerIsStale(directory: string, owner: LeaseOwner): Promise { + if (owner.hostname === this.hostname) { + let serverAlive = (this.options.isPidAlive ?? isPidAlive)(owner.pid) + if (owner.processStart) { + const currentStart = await readProcessStart(owner.pid) + if (currentStart) serverAlive = currentStart === owner.processStart + } + if (serverAlive) return false + const launches = await readLaunchGenerations(directory, owner.leaseToken) + if (launches.kind === "unknown") return false + const platform = this.options.platform ?? process.platform + if (launches.kind === "absent") return true + const identities: ProcessIdentity[] = [] + for (const launch of launches.launches) { + let tokenAlive: boolean | undefined + try { + tokenAlive = this.options.isLaunchTokenAlive + ? this.options.isLaunchTokenAlive(launch.token) + : launchTokenIsAlive(launch.token, platform) + } catch { + tokenAlive = undefined + } + if (tokenAlive === true) return false + if (!launch.complete) { + if (platform === "win32" || tokenAlive === undefined) return false + continue + } + identities.push(...launch.identities!) + } + if (launches.malformed && (launches.malformed.failClosed || platform === "win32" || + Date.now() - launches.malformed.modifiedAt < this.staleMs)) return false + return identities.length === 0 || this.persistedProcessesAreGone(identities) + } + return false + } + + private persistedProcessesAreGone(identities: ProcessIdentity[]): boolean { + if (identities.length === 0) return true + if (this.options.isProcessIdentityAlive) { + for (const identity of identities) { + try { + if (this.options.isProcessIdentityAlive(identity) !== false) return false + } catch { + return false + } + } + return true + } + const platform = this.options.platform ?? process.platform + const snapshot = platform === "win32" + ? probeWindowsProcesses(spawnSync, 1_000) + : probePosixProcesses(spawnSync, 1_000, platform, { pids: identities.map(({ pid }) => pid) }) + return snapshot.ok && identities.every((identity) => !sameProcess(identity, snapshot.processes.get(identity.pid))) + } +} + +async function publishOwner(directory: string, owner: LeaseOwner): Promise { + const temporary = path.join(directory, `.owner.${owner.leaseToken}.tmp`) + const destination = path.join(directory, "owner") + try { + await fs.mkdir(temporary) + const file = await fs.open(path.join(temporary, OWNER_FILE), "wx", 0o600) + try { + await file.writeFile(JSON.stringify(owner), "utf8") + await file.sync() + } finally { + await file.close() + } + await fs.writeFile(path.join(temporary, "heartbeat"), randomUUID(), { encoding: "utf8", flag: "wx", mode: 0o600 }) + await fs.rename(temporary, destination) + return true + } catch (error) { + if (hasCode(error, "EEXIST") || hasCode(error, "ENOTEMPTY") || hasCode(error, "EPERM")) return false + throw error + } finally { + await fs.rm(temporary, { recursive: true, force: true }).catch(() => undefined) + } +} + +async function readOwner(directory: string, ownerDirectory = "owner"): Promise { + try { + const [serialized, heartbeat] = await Promise.all([ + fs.readFile(path.join(directory, ownerDirectory, OWNER_FILE), "utf8"), + fs.readFile(path.join(directory, ownerDirectory, "heartbeat"), "utf8"), + ]) + const owner = JSON.parse(serialized) as Partial + if (owner.version !== 1 || !safeToken(owner.managerToken) || !safeToken(owner.leaseToken) || + !Number.isInteger(owner.pid) || Number(owner.pid) <= 0 || typeof owner.hostname !== "string" || + typeof owner.workspacePath !== "string") return undefined + return { owner: owner as LeaseOwner, serialized, heartbeat } + } catch (error) { + if (hasCode(error, "ENOENT") || error instanceof SyntaxError) return undefined + throw error + } +} + +async function heartbeatOwner(directory: string, owner: LeaseOwner): Promise { + const current = await readOwner(directory) + if (current?.owner.leaseToken !== owner.leaseToken) return false + const temporary = path.join(directory, `.heartbeat.${owner.leaseToken}.${randomUUID()}.tmp`) + await fs.writeFile(temporary, randomUUID(), { encoding: "utf8", flag: "wx", mode: 0o600 }) + try { + if ((await readOwner(directory))?.owner.leaseToken !== owner.leaseToken) return false + await fs.rename(temporary, path.join(directory, "owner", "heartbeat")) + } finally { + await fs.rm(temporary, { force: true }).catch(() => undefined) + } + return (await readOwner(directory))?.owner.leaseToken === owner.leaseToken +} + +async function retireOwner(directory: string, observed: ObservedOwner): Promise { + const current = await readOwner(directory) + if (!sameObservedOwner(current, observed)) return false + const retired = `retired.${observed.owner.leaseToken}` + try { + // ponytail: generation tombstones are tiny and prevent a stale reclaimer from ever targeting a successor. + await fs.rename(path.join(directory, "owner"), path.join(directory, retired)) + const moved = await readOwner(directory, retired) + if (!sameObservedOwner(moved, observed)) { + await fs.rename(path.join(directory, retired), path.join(directory, "owner")).catch(() => undefined) + return false + } + return true + } catch (error) { + if (["ENOENT", "EEXIST", "ENOTEMPTY", "EPERM"].some((code) => hasCode(error, code))) return false + throw error + } +} + +function sameObservedOwner(left: ObservedOwner | undefined, right: ObservedOwner): boolean { + return left?.serialized === right.serialized + && left.owner.leaseToken === right.owner.leaseToken + && left.heartbeat === right.heartbeat +} + +function safeToken(value: unknown): value is string { + return typeof value === "string" && /^[A-Za-z0-9_-]+$/.test(value) +} + +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return !hasCode(error, "ESRCH") + } +} + +async function readProcessStart(pid: number): Promise { + if (process.platform !== "linux") return undefined + try { + const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8") + return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19] + } catch { + return undefined + } +} + +async function writeLaunchGeneration(directory: string, leaseToken: string, launch: LaunchGeneration, + replace = false): Promise { + if (launch.complete && (!launch.identities?.length || launch.identities.some((identity) => !validProcessIdentity(identity)))) { + throw new Error("A complete process identity generation requires at least one valid identity") + } + const temporary = path.join(directory, `.launch.${leaseToken}.${launch.generation}.${randomUUID()}.tmp`) + const destination = launchGenerationPath(directory, leaseToken, launch.generation) + const handle = await fs.open(temporary, "wx", 0o600) + try { + await handle.writeFile(JSON.stringify(launch), "utf8") + await handle.sync() + } finally { + await handle.close() + } + try { + if (!replace && await fileExists(destination)) throw Object.assign(new Error("Launch generation already exists"), { code: "EEXIST" }) + await fs.rename(temporary, destination) + } finally { + await fs.rm(temporary, { force: true }).catch(() => undefined) + } +} + +async function removeLaunchGeneration(directory: string, leaseToken: string, generation: string): Promise { + await fs.rm(launchGenerationPath(directory, leaseToken, generation)) +} + +type LaunchGenerationObservation = + | { kind: "absent" } + | { kind: "observed"; launches: LaunchGeneration[]; malformed?: { modifiedAt: number; failClosed: boolean } } + | { kind: "unknown" } + +async function readLaunchGenerations(directory: string, leaseToken: string): Promise { + let entries: string[] + try { + entries = await fs.readdir(directory) + } catch (error) { + if (hasCode(error, "ENOENT")) return { kind: "absent" } + return { kind: "unknown" } + } + const legacyLaunch = `launch.${leaseToken}.json` + const launchPrefix = `launch.${leaseToken}.` + const processPrefix = `process.${leaseToken}.` + const launchEntries = entries.filter((entry) => entry === legacyLaunch || + (entry.startsWith(launchPrefix) && entry.endsWith(".json"))) + const processEntries = entries.filter((entry) => entry.startsWith(processPrefix) && entry.endsWith(".json")) + if (launchEntries.length === 0 && processEntries.length === 0) return { kind: "absent" } + + const launches: LaunchGeneration[] = [] + const malformedGenerationPaths: string[] = [] + let legacyMalformed = processEntries.length > 0 && !launchEntries.includes(legacyLaunch) + for (const entry of launchEntries) { + const launchPath = path.join(directory, entry) + let serialized: string + let value: unknown + try { + serialized = await fs.readFile(launchPath, "utf8") + value = JSON.parse(serialized) + } catch (error) { + if (!(error instanceof SyntaxError)) return { kind: "unknown" } + if (entry === legacyLaunch) legacyMalformed = true + else malformedGenerationPaths.push(launchPath) + continue + } + + if (entry === legacyLaunch) { + const token = value && typeof value === "object" ? (value as { token?: unknown }).token : undefined + const identities: ProcessIdentity[] = [] + let complete = safeToken(token) && processEntries.length > 0 + for (const processEntry of processEntries) { + try { + const processSerialized = await fs.readFile(path.join(directory, processEntry), "utf8") + const identity = JSON.parse(processSerialized) as unknown + const digest = createHash("sha256").update(processSerialized).digest("hex") + if (!validProcessIdentity(identity) || processEntry !== `process.${leaseToken}.${digest}.json`) complete = false + else identities.push(identity) + } catch { + complete = false + } + } + if (complete && safeToken(token)) launches.push({ version: 1, generation: "legacy", token, complete: true, identities }) + else legacyMalformed = true + continue + } + + const launch = value as LaunchGeneration + if (validLaunchGeneration(launch) && launchGenerationPath(directory, leaseToken, launch.generation) === launchPath) { + launches.push(launch) + } else { + malformedGenerationPaths.push(launchPath) + } + } + + if (legacyMalformed) { + return { kind: "observed", launches, malformed: { modifiedAt: Date.now(), failClosed: true } } + } + if (malformedGenerationPaths.length === 0) return { kind: "observed", launches } + try { + const stats = await Promise.all([...malformedGenerationPaths, path.join(directory, "owner", "heartbeat")] + .map((file) => fs.stat(file))) + return { + kind: "observed", + launches, + malformed: { modifiedAt: Math.max(...stats.map((stat) => stat.mtimeMs)), failClosed: false }, + } + } catch { + return { kind: "unknown" } + } +} + +function validLaunchGeneration(value: LaunchGeneration): boolean { + return value?.version === 1 && safeToken(value.generation) && safeToken(value.token) && typeof value.complete === "boolean" && + (!value.complete || Boolean(value.identities?.length && value.identities.every(validProcessIdentity))) +} + +function launchGenerationPath(directory: string, leaseToken: string, generation: string): string { + return path.join(directory, `launch.${leaseToken}.${generation}.json`) +} + +async function fileExists(file: string): Promise { + try { + await fs.access(file) + return true + } catch (error) { + if (hasCode(error, "ENOENT")) return false + throw error + } +} + +function validProcessIdentity(value: unknown): value is ProcessIdentity { + if (!value || typeof value !== "object") return false + const identity = value as Partial + return Number.isInteger(identity.pid) && Number(identity.pid) > 0 && + Number.isInteger(identity.parentPid) && Number(identity.parentPid) >= 0 && + Number.isInteger(identity.groupId) && Number(identity.groupId) > 0 && + typeof identity.startTime === "string" && identity.startTime.length > 0 && + (identity.bootId === undefined || typeof identity.bootId === "string") && + (identity.startOrder === undefined || typeof identity.startOrder === "string") +} + +function launchTokenIsAlive(token: string, platform: NodeJS.Platform): boolean | undefined { + if (!token) return undefined + if (platform === "win32") return undefined + const snapshot = probeLaunchCleanupToken(spawnSync, token, 1_000, undefined, platform) + return snapshot.ok ? snapshot.processes.size > 0 : undefined +} + +function hasCode(error: unknown, code: string): boolean { + return error instanceof Error && "code" in error && error.code === code +} diff --git a/packages/server/src/workspaces/runtime.test.ts b/packages/server/src/workspaces/runtime.test.ts index 54fb92a9e..5e7387321 100644 --- a/packages/server/src/workspaces/runtime.test.ts +++ b/packages/server/src/workspaces/runtime.test.ts @@ -52,7 +52,12 @@ const token = (rows: Array<[number, number, number, string]>, signal: boolean, b const isToken = (args: readonly string[]) => args.includes("codenomad-token-cleanup") const isSignal = (args: readonly string[]) => isToken(args) && (args.includes("TERM") || args.includes("KILL")) const isGuarded = (args: readonly string[]) => !isToken(args) && args.some((arg) => arg.includes("guarded-signal") || arg.includes("CODENOMAD_RESULT")) -async function harness(options: WorkspaceRuntimeOptions & { binary?: string; output?: string; report?: boolean } = {}) { +async function harness(options: WorkspaceRuntimeOptions & { + binary?: string + output?: string + report?: boolean + persistProcessIdentities?: (identities: import("./process-identity").ProcessIdentity[]) => Promise +} = {}) { const child = new FakeChild() const timers = new ManualTimers() const calls: Call[] = [] @@ -75,7 +80,10 @@ async function harness(options: WorkspaceRuntimeOptions & { binary?: string; out }) const abort = new AbortController() const folder = platform === "win32" && process.platform !== "win32" ? `/${process.cwd()}` : process.cwd() - const launch = runtime.launch({ workspaceId: "w", folder, binaryPath: options.binary ?? "opencode", signal: abort.signal }) + const launch = runtime.launch({ + workspaceId: "w", folder, binaryPath: options.binary ?? "opencode", signal: abort.signal, + persistProcessIdentities: options.persistProcessIdentities, + }) if (options.report !== false) { queueMicrotask(() => child.stdout.write(options.output ?? "opencode server listening on http://127.0.0.1:4321\n")) await launch @@ -92,6 +100,128 @@ describe("workspace runtime lifecycle contracts", () => { assert.deepEqual(launchCall?.args.slice(-1), ["4242"]) }) + it("persists a Windows wrapper and its launch descendants before publishing the port", async () => { + let persisted: number[] = [] + let captures = 0 + await harness({ + platform: "win32", + binary: "opencode.cmd", + persistProcessIdentities: async (identities) => { persisted = identities.map(({ pid }) => pid) }, + spawnSync: ((command: string) => command === "powershell.exe" + ? result(windows(++captures === 1 + ? [[4242, 1, "wrapper-start"]] + : [[4242, 1, "wrapper-start"], [5000, 4242, "child-start"]])) + : result()) as unknown as Command, + }) + assert.deepEqual(persisted, [4242, 5000]) + }) + + it("fails closed and cleans the known Windows tree when readiness-time CIM capture fails", async () => { + const child = new FakeChild() + const calls: Call[] = [] + let captures = 0 + let persisted = false + const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), { + platform: "win32", + spawn: (() => child as unknown as ChildProcess) as typeof import("node:child_process").spawn, + spawnSync: ((command: string, args: readonly string[]) => { + calls.push({ command, args: [...args] }) + if (command === "powershell.exe") { + return ++captures === 1 + ? result(windows([[4242, 1, "wrapper-start"]])) + : result("", 1, "second CIM capture failed") + } + return result() + }) as unknown as Command, + }) + const folder = process.platform === "win32" ? process.cwd() : `/${process.cwd()}` + const launch = runtime.launch({ + workspaceId: "w", folder, binaryPath: "opencode.cmd", + persistProcessIdentities: async () => { persisted = true }, + }) + child.stdout.write("opencode server listening on http://127.0.0.1:4321\n") + + await assert.rejects(launch, (error: unknown) => + error instanceof WorkspaceRuntimeIdentityCaptureError && /readiness-time Windows identity discovery failed/.test(error.message)) + assert.equal(persisted, false) + assert.ok(calls.some(({ command, args }) => command === "taskkill.exe" && args.includes("/T"))) + }) + + it("retries direct Windows cleanup after a transient readiness-time CIM failure", async () => { + const child = new FakeChild() + const timers = new ManualTimers() + const calls: Call[] = [] + let captures = 0 + let cimAvailable = false + let alive = true + const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), { + platform: "win32", gracefulStopTimeoutMs: 10, forcedStopTimeoutMs: 10, + setTimeout: timers.set, clearTimeout: timers.clear, + spawn: (() => child as unknown as ChildProcess) as typeof import("node:child_process").spawn, + spawnSync: ((command: string, args: readonly string[]) => { + calls.push({ command, args: [...args] }) + if (command !== "powershell.exe") return result() + if (isGuarded(args)) { + if (!cimAvailable) return result("", 1, "CIM unavailable") + alive = false + return result("CODENOMAD_TARGET|4242|1|0|direct-start||100\nCODENOMAD_RESULT|1||1") + } + captures += 1 + if (captures === 1) return result(windows([[4242, 1, "direct-start"]])) + if (!cimAvailable) return result("", 1, "CIM unavailable") + return result(windows(alive ? [[4242, 1, "direct-start"]] : [[1, 0, "system-start"]])) + }) as unknown as Command, + }) + const folder = process.platform === "win32" ? process.cwd() : `/${process.cwd()}` + const launch = runtime.launch({ workspaceId: "w", folder, binaryPath: "opencode.exe" }) + child.stdout.write("opencode server listening on http://127.0.0.1:4321\n") + + await assert.rejects(launch, WorkspaceRuntimeIdentityCaptureError) + const firstCleanup = runtime.stop("w") + timers.run(); timers.run() + await assert.rejects(firstCleanup, WorkspaceStopTimeoutError) + + cimAvailable = true + await runtime.stop("w") + assert.equal(calls.some(({ command }) => command === "taskkill.exe"), false) + assert.equal((runtime as unknown as { processes: Map }).processes.size, 0) + }) + + it("retains unknown Windows ownership when the leader exits before identity recapture", async () => { + const child = new FakeChild() + const timers = new ManualTimers() + let captures = 0 + let cimAvailable = false + const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), { + platform: "win32", gracefulStopTimeoutMs: 10, forcedStopTimeoutMs: 10, + setTimeout: timers.set, clearTimeout: timers.clear, + spawn: (() => child as unknown as ChildProcess) as typeof import("node:child_process").spawn, + spawnSync: ((_command: string, args: readonly string[]) => { + if (++captures === 1) return result(windows([[4242, 1, "direct-start"]])) + if (!cimAvailable) return result("", 1, "CIM unavailable") + return isGuarded(args) + ? result("CODENOMAD_RESULT|0||0") + : result(windows([[5000, 4242, "child-start"]])) + }) as unknown as Command, + }) + const folder = process.platform === "win32" ? process.cwd() : `/${process.cwd()}` + const launch = runtime.launch({ workspaceId: "w", folder, binaryPath: "opencode.exe" }) + child.stdout.write("opencode server listening on http://127.0.0.1:4321\n") + + await assert.rejects(launch, WorkspaceRuntimeIdentityCaptureError) + const failedCleanup = runtime.stop("w") + timers.run(); timers.run() + await assert.rejects(failedCleanup, WorkspaceStopTimeoutError) + + child.exit(1) + cimAvailable = true + const retry = runtime.stop("w") + timers.run(); timers.run() + await assert.rejects(retry, (error: unknown) => + error instanceof WorkspaceStopTimeoutError && /cannot prove exact launch ownership/.test(error.message)) + assert.equal((runtime as unknown as { processes: Map }).processes.size, 1) + }) + it("cancels before spawn and while waiting for a port without losing retryable cleanup", async () => { let spawned = false const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), { @@ -241,15 +371,18 @@ describe("workspace runtime lifecycle contracts", () => { const calls: Call[] = [] const h = await harness({ platform: "win32", binary: "opencode.cmd", spawnSync: ((command: string, args: readonly string[]) => { calls.push({ command, args: [...args] }) + if (command === "powershell.exe") return result(windows([[4242, 1, "wrapper-start"]])) return available ? result() : result("", 1, "taskkill unavailable") }) as unknown as Command }) const first = h.runtime.stop("w"); h.timers.run(); h.timers.run() await assert.rejects(first, (error: unknown) => error instanceof WorkspaceStopTimeoutError && /\/T \/F failed/.test(error.message)) - assert.deepEqual(calls.map(({ args }) => args), [["/PID", "4242", "/T"], ["/PID", "4242", "/T", "/F"]]) + assert.deepEqual(calls.filter(({ command }) => command === "taskkill.exe").map(({ args }) => args), + [["/PID", "4242", "/T"], ["/PID", "4242", "/T", "/F"]]) available = true const retry = h.runtime.stop("w"); h.child.exit(); await retry - const exited = await harness({ platform: "win32", binary: "opencode.cmd", spawnSync: (() => result("", 1, "taskkill unavailable")) as unknown as Command }) + const exited = await harness({ platform: "win32", binary: "opencode.cmd", spawnSync: ((command: string) => + command === "powershell.exe" ? result(windows([[4242, 1, "wrapper-start"]])) : result("", 1, "taskkill unavailable")) as unknown as Command }) const incomplete = exited.runtime.stop("w"); exited.child.exit(1) await assert.rejects(incomplete, WorkspaceWindowsTreeCleanupIncompleteError) await assert.rejects(exited.runtime.stop("w"), WorkspaceWindowsTreeCleanupIncompleteError) diff --git a/packages/server/src/workspaces/runtime.ts b/packages/server/src/workspaces/runtime.ts index 41adab8cb..f1dd1b283 100644 --- a/packages/server/src/workspaces/runtime.ts +++ b/packages/server/src/workspaces/runtime.ts @@ -48,6 +48,8 @@ interface LaunchOptions { logLevel?: string onExit?: (info: ProcessExitInfo) => void signal?: AbortSignal + cleanupToken?: string + persistProcessIdentities?: (identities: ProcessIdentity[]) => Promise } export interface ProcessExitInfo { @@ -169,13 +171,14 @@ export class WorkspaceRuntime { port: number exitPromise: Promise getLastOutput: () => string + processIdentity?: ProcessIdentity }> { options.signal?.throwIfAborted() this.validateFolder(options.folder) const logLevel = typeof options.logLevel === "string" ? options.logLevel.toUpperCase() : "DEBUG" const args = ["serve", "--port", "0", "--print-logs", "--log-level", logLevel] - const cleanupToken = randomBytes(32).toString("hex") + const cleanupToken = options.cleanupToken ?? randomBytes(32).toString("hex") const env = { ...process.env, ...(options.environment ?? {}), [LAUNCH_CLEANUP_TOKEN_ENV]: cleanupToken } let exitResolve: ((info: ProcessExitInfo) => void) | null = null @@ -268,7 +271,7 @@ export class WorkspaceRuntime { : {}), } this.processes.set(options.workspaceId, managed) - if (spec.processKind === "posix" || spec.processKind === "wsl" || spec.processKind === "windows-direct") { + if (spec.processKind === "posix" || spec.processKind === "wsl" || spec.processKind === "windows-direct" || spec.processKind === "windows-wrapper") { const launchSnapshot = child.pid ? this.platform === "win32" ? probeWindowsProcesses(this.spawnCommand, this.stopCommandTimeoutMs) @@ -295,6 +298,11 @@ export class WorkspaceRuntime { for (const identity of launchSnapshot.ok ? launchSnapshot.processes.values() : [launchLeader]) { if (identity.groupId === launchLeader.groupId) managed.targets!.members.set(identity.pid, identity) } + if (this.platform === "win32" && launchSnapshot.ok) { + for (const identity of descendantsOf(launchSnapshot.processes, launchLeader.pid)) { + managed.targets!.members.set(identity.pid, identity) + } + } } let stdoutBuffer = "" @@ -302,6 +310,7 @@ export class WorkspaceRuntime { let portFound = false let pendingPort: number | null = null let launchSettled = false + let launchPersistenceStarted = false const cancelLaunch = () => { if (launchSettled) return launchSettled = true @@ -401,19 +410,48 @@ export class WorkspaceRuntime { child.on("error", handleError) child.on("exit", handleExit) - const resolveLaunchIfIdentified = () => { - if (launchSettled || pendingPort === null) return + const resolveLaunchIfIdentified = async () => { + if (launchSettled || launchPersistenceStarted || pendingPort === null) return if (managed.wsl && (!managed.wsl.linuxPid || !managed.wsl.linuxPgid || !managed.wsl.leaderStartTime || !managed.wsl.bootId)) { return } + launchPersistenceStarted = true portFound = true - launchSettled = true stopWarningTimer() + if (this.platform === "win32" && managed.targets?.leader) { + const snapshot = probeWindowsProcesses(this.spawnCommand, this.stopCommandTimeoutMs) + const currentLeader = snapshot.ok ? snapshot.processes.get(managed.targets.leader.pid) : undefined + if (!snapshot.ok || !currentLeader || !sameProcess(managed.targets.leader, currentLeader)) { + launchSettled = true + const detail = !snapshot.ok + ? `readiness-time Windows identity discovery failed: ${snapshot.error}` + : `spawned PID ${managed.targets.leader.pid} changed or disappeared before readiness-time identity capture` + this.beginFailedLaunchCleanup(options.workspaceId, managed) + reject(new WorkspaceRuntimeIdentityCaptureError(options.workspaceId, detail)) + return + } + managed.targets.members.clear() + managed.targets.members.set(currentLeader.pid, currentLeader) + for (const identity of descendantsOf(snapshot.processes, currentLeader.pid)) managed.targets.members.set(identity.pid, identity) + } + try { + await options.persistProcessIdentities?.([...managed.targets!.members.values()]) + } catch (error) { + launchSettled = true + this.beginFailedLaunchCleanup(options.workspaceId, managed) + reject(new WorkspaceRuntimeIdentityCaptureError( + options.workspaceId, + `launch-tree identity persistence failed: ${error instanceof Error ? error.message : String(error)}`, + )) + return + } + if (launchSettled) return + launchSettled = true options.signal?.removeEventListener("abort", cancelLaunch) managed.cancelLaunch = undefined child.removeListener("error", handleError) this.logger.info({ workspaceId: options.workspaceId, port: pendingPort }, "Workspace runtime allocated port") - resolve({ pid: child.pid!, port: pendingPort, exitPromise, getLastOutput }) + resolve({ pid: child.pid!, port: pendingPort, exitPromise, getLastOutput, processIdentity: managed.targets?.leader }) } const failWslIdentityCapture = (detail: string) => { @@ -461,7 +499,7 @@ export class WorkspaceRuntime { }, "Captured WSL OpenCode process identity", ) - resolveLaunchIfIdentified() + void resolveLaunchIfIdentified() } else { failWslIdentityCapture("WSL launcher returned an incomplete Linux PID identity") } @@ -482,7 +520,7 @@ export class WorkspaceRuntime { if (managed.wsl && (!managed.wsl.leaderStartTime || !managed.wsl.bootId)) { failWslIdentityCapture("WSL process reported a port before its Linux identity") } else { - resolveLaunchIfIdentified() + void resolveLaunchIfIdentified() } } } @@ -617,6 +655,7 @@ export class WorkspaceRuntime { } if (this.platform === "win32" && !managed.wsl && leaderMatches && leader) { for (const descendant of descendantsOf(snapshot.processes, leader.pid)) target.members.set(descendant.pid, descendant) + if (managed.processKind === "windows-direct") managed.identityCaptureFailed = false } const aliveMembers = Array.from(target.members.values()).filter((identity) => sameProcess(identity, snapshot.processes.get(identity.pid)), @@ -647,6 +686,8 @@ export class WorkspaceRuntime { const sendStopSignal = (signal: NodeJS.Signals) => { if (!pid) failures.push(`${signal} was not sent because the process PID is unavailable`) if (pid && wrapperExited() && this.platform !== "linux" && this.platform !== "win32") refreshTargets() + if (pid && managed.identityCaptureFailed && this.platform === "win32" && !managed.wsl && + managed.processKind === "windows-direct") refreshTargets() let signaledOwnedGroup = false if (pid && managed.identityCaptureFailed && this.platform !== "linux" && this.platform !== "win32" && !wrapperExited()) { const result = signalOwnedPosixProcessGroup(this.spawnCommand, pid, signal, this.stopCommandTimeoutMs) diff --git a/packages/server/src/workspaces/spawn.ts b/packages/server/src/workspaces/spawn.ts index 52ec18d79..6d7d10daa 100644 --- a/packages/server/src/workspaces/spawn.ts +++ b/packages/server/src/workspaces/spawn.ts @@ -16,6 +16,7 @@ const CODENOMAD_PLUGIN_FILE_SPEC_REGEX = new RegExp( const WSL_PATH_ENV_KEYS = new Set(["NODE_EXTRA_CA_CERTS", WSL_PLUGIN_PATH_ENV]) const WINDOWS_DIRECT_EXTENSIONS = new Set([".com", ".exe"]) const DEFAULT_WINDOWS_PATHEXT = ".COM;.EXE;.BAT;.CMD" +const WINDOWS_CMD_META_CHARS = /([()\][%!^"`<>&|;, *?])/g const WINDOWS_SHELL_NAMES = new Set([ "bash", "bash.exe", @@ -104,13 +105,14 @@ export function buildWindowsSpawnSpec(binaryPath: string, args: string[], option const comspec = getWindowsEnvironmentValue(options.env, "COMSPEC") ?? getWindowsEnvironmentValue(process.env, "COMSPEC") ?? "cmd.exe" - // cmd.exe requires the full command as a single string. - // Using the ""